From bb1f1d3faa35734f945afe2c479e5e3de990af24 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Tue, 20 Jan 2015 16:31:36 +0000 Subject: [PATCH 1/4] Merged manually from caxapniy/s3fs-fuse/tree/1.77v4merge for signature v4 - #102 --- src/common.h | 3 + src/common_auth.cpp | 23 ++ src/curl.cpp | 774 +++++++++++++++++++++++++++++++++++++++++++ src/curl.h | 8 + src/openssl_auth.cpp | 86 +++++ src/s3fs.cpp | 9 + src/s3fs_auth.h | 8 + src/s3fs_util.cpp | 5 + src/string_util.cpp | 58 ++++ src/string_util.h | 7 + 10 files changed, 981 insertions(+) diff --git a/src/common.h b/src/common.h index dfc0343..cb7a4ea 100644 --- a/src/common.h +++ b/src/common.h @@ -94,6 +94,9 @@ extern std::string service_path; extern std::string host; extern std::string bucket; extern std::string mount_prefix; +#ifndef SIGV3 +extern std::string endpoint; +#endif #endif // S3FS_COMMON_H_ diff --git a/src/common_auth.cpp b/src/common_auth.cpp index 9dd4828..14b7e0f 100644 --- a/src/common_auth.cpp +++ b/src/common_auth.cpp @@ -102,6 +102,29 @@ string s3fs_md5sum(int fd, off_t start, ssize_t size) return string(md5); } +#ifndef SIGV3 +string s3fs_sha256sum(int fd, off_t start, ssize_t size) +{ + size_t digestlen = get_sha256_digest_length(); + char sha256[2 * digestlen + 1]; + char hexbuf[3]; + unsigned char* sha256hex; + + if(NULL == (sha256hex = s3fs_sha256hexsum(fd, start, size))){ + return string(""); + } + + memset(sha256, 0, 2 * digestlen + 1); + for(size_t pos = 0; pos < digestlen; pos++){ + snprintf(hexbuf, 3, "%02x", sha256hex[pos]); + strncat(sha256, hexbuf, 2); + } + free(sha256hex); + + return string(sha256); +} +#endif + /* * Local variables: * tab-width: 4 diff --git a/src/curl.cpp b/src/curl.cpp index a026b02..14b51d6 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -1742,6 +1742,7 @@ int S3fsCurl::RequestPerform(void) // @param date e.g., get_date() // @param resource e.g., "/pub" // +#ifdef SIGV3 string S3fsCurl::CalcSignature(string method, string strMD5, string content_type, string date, string resource) { string Signature; @@ -1789,6 +1790,131 @@ string S3fsCurl::CalcSignature(string method, string strMD5, string content_type return Signature; } +#else +// NOT SIGV3 +string S3fsCurl::CalcSignaturev2(string method, string strMD5, string content_type, string date, string resource) +{ + string Signature; + string StringToSign; + + if(0 < S3fsCurl::IAM_role.size()){ + if(!S3fsCurl::CheckIAMCredentialUpdate()){ + DPRN("Something error occurred in checking IAM credential."); + return Signature; // returns empty string, then it occures error. + } + requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str()); + } + + StringToSign += method + "\n"; + StringToSign += strMD5 + "\n"; // md5 + StringToSign += content_type + "\n"; + StringToSign += date + "\n"; + for(curl_slist* headers = requestHeaders; headers; headers = headers->next){ + if(0 == strncmp(headers->data, "x-amz", 5)){ + StringToSign += headers->data; + StringToSign += "\n"; + } + } + StringToSign += resource; + + const void* key = S3fsCurl::AWSSecretAccessKey.data(); + int key_len = S3fsCurl::AWSSecretAccessKey.size(); + const unsigned char* sdata = reinterpret_cast(StringToSign.data()); + int sdata_len = StringToSign.size(); + unsigned char* md = NULL; + unsigned int md_len = 0;; + + s3fs_HMAC(key, key_len, sdata, sdata_len, &md, &md_len); + + char* base64; + if(NULL == (base64 = s3fs_base64(md, md_len))){ + free(md); + return string(""); // ENOMEM + } + free(md); + + Signature = base64; + free(base64); + + return Signature; +} + +string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string query_string , string date2, + string canonical_headers, string payload_hash, string signed_headers, string date3) +{ + string Signature, StringCQ, StringToSign; + string uriencode; + + if(0 < S3fsCurl::IAM_role.size()){ + if(!S3fsCurl::CheckIAMCredentialUpdate()){ + DPRN("Something error occurred in checking IAM credential."); + return Signature; // returns empty string, then it occures error. + } + requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str()); + } + + uriencode = urlEncode(canonical_uri); + StringCQ = method + "\n"; + if(0 == strcmp(method.c_str(),"HEAD") || 0 == strcmp(method.c_str(),"PUT") || 0 == strcmp(method.c_str(),"DELETE")){ + StringCQ += uriencode + "\n" + query_string + "\n"; + }else if (0 == strcmp(method.c_str(), "GET") && 0 == strcmp(uriencode.c_str(), "")) { + StringCQ +="/\n\n"; + }else if (0 == strcmp(method.c_str(), "GET") && 0 == strncmp(uriencode.c_str(), "/",1)) { + StringCQ += uriencode +"\n\n"; + }else if (0 == strcmp(method.c_str(), "GET") && 0 != strncmp(uriencode.c_str(), "/",1)) { + StringCQ += "/\n" + urlEncode2(canonical_uri) +"\n"; + }else if (0 == strcmp(method.c_str(), "POST")) { + StringCQ += uriencode +"\n" + query_string +"\n"; + } + StringCQ += canonical_headers + "\n"; + StringCQ += signed_headers + "\n"; + StringCQ += payload_hash; + unsigned char * cRequest = (unsigned char *)StringCQ.c_str(); + unsigned int cRequest_len= StringCQ.size(); +// DPRN("SHUNDEBUGXXXPUT: %s", cRequest); + char kSecret[128]; + unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest = NULL; + unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0; + char hexsRequest[64]; + int kSecret_len = snprintf(kSecret, sizeof(kSecret), "AWS4%s", S3fsCurl::AWSSecretAccessKey.c_str()); + unsigned int i; + + s3fs_HMAC(kSecret, kSecret_len, (unsigned char*)date2.data(), date2.size(), &kDate, &kDate_len); + s3fs_HMAC(kDate, kDate_len, (unsigned char *)endpoint.c_str(), endpoint.size(), &kRegion, &kRegion_len); + s3fs_HMAC(kRegion, kRegion_len, (unsigned char *)"s3", sizeof("s3")-1, &kService, &kService_len); + s3fs_HMAC(kService, kService_len, (unsigned char *)"aws4_request", sizeof("aws4_request")-1, &kSigning, &kSigning_len); + + s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len); + //for (i=0;i < sRequest_len;i++) printf("%02x", sRequest[i]); + + for (i=0;i < sRequest_len;i++) sprintf(hexsRequest+(i*2), "%02x", sRequest[i]); + + StringToSign = "AWS4-HMAC-SHA256\n"; + StringToSign += date3+"\n"; + StringToSign += date2+"/" + endpoint + "/s3/aws4_request\n"; + StringToSign += hexsRequest; + + + unsigned char* cscope = (unsigned char*)StringToSign.c_str(); + unsigned int cscope_len = StringToSign.size(); + unsigned char* md = NULL; + unsigned int md_len = 0; + char hexSig[64]; + s3fs_HMAC(kSigning, kSigning_len, cscope, cscope_len, &md, &md_len); + for (i=0; i < md_len; i++) sprintf(hexSig+(i*2), "%02x", md[i]); + + Signature = hexSig; + + return Signature; +} + +string S3fsCurl::CalcSignature(string method, string canonical_uri, string date2, + string canonical_headers, string payload_hash, string signed_headers, string date3) +{ + return CalcSignatureReal(method, canonical_uri, "", date2, canonical_headers, payload_hash, signed_headers, date3); +} +#endif + // XML in BodyData has UploadId, Parse XML body for UploadId bool S3fsCurl::GetUploadId(string& upload_id) { @@ -1852,6 +1978,7 @@ int S3fsCurl::DeleteRequest(const char* tpath) requestHeaders = NULL; responseHeaders.clear(); +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); @@ -1861,6 +1988,30 @@ int S3fsCurl::DeleteRequest(const char* tpath) string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("DELETE", "", "", date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=tpath; + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ + + payload_hash + "\nx-amz-date:" + date3 + "\n"; + string signed_headers = "host;x-amz-content-sha256;x-amz-date"; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + + "/" + date2 + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("DELETE", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); @@ -1966,6 +2117,7 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* responseHeaders.clear(); // requestHeaders +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); @@ -1984,6 +2136,31 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("HEAD", "", "", date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=tpath; + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ + + payload_hash + "\nx-amz-date:" + date3 + "\n"; + string signed_headers = "host;x-amz-content-sha256;x-amz-date"; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + + //DPRN("SHUNDEBUG3SIGN %s", string("Date:"+date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("HEAD", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_NOBODY, true); // HEAD @@ -2081,6 +2258,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) bodydata = new BodyData(); // Make request headers +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); @@ -2108,6 +2286,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) } } } + // "x-amz-acl", rrs, sse requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); if(S3fsCurl::is_use_rrs){ @@ -2131,6 +2310,93 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("PUT", "", ContentType, date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=tpath; + //string payload_hash = s3fs_sha256sum(fd, 0, -1); + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + string ContentType; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; + string signed_headers = "host"; + // "x-amz-acl" + requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); + canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n"; + signed_headers += ";x-amz-acl"; + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; + signed_headers += ";x-amz-content-sha256"; + + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + canonical_headers += "x-amz-date:" + date3 + "\n"; + signed_headers += ";x-amz-date"; + + for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ + string key = (*iter).first; + string value = (*iter).second; + if(0 == strcasecmp(key.c_str(), "Content-Type")){ + ContentType = value; + requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + canonical_headers.insert(0, "content-type:" + value + "\n"); + signed_headers.insert(0, "content-type;"); + }else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ + // not set value, but after set it. + }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ + requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + canonical_headers += key + ":" + value + "\n"; + signed_headers += ";" + key; + }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){ + requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n")); + signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key)); + }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ + // skip this header, because this header is specified after logic. + }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ + // skip this header, because this header is specified with "x-amz-...-customer-key-md5". + }else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ + // Only copy mode. + if(!AddSseKeyRequestHead(value, is_copy)){ + DPRNNN("Failed to insert sse(-c) header."); + } + } + } + + // rrs, sse + if(S3fsCurl::is_use_sse){ + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); + canonical_headers += "x-amz-server-side-encryption:AES256\n"; + signed_headers += ";x-amz-server-side-encryption"; + }else if(S3fsCurl::IsSseCustomMode()){ + string md5; + if(!AddSseKeyRequestHead(md5, false)){ + DPRNNN("Failed to insert sse(-c) header."); + } + } + if(S3fsCurl::is_use_rrs){ + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); + canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n"; + signed_headers += ";x-amz-storage-class"; + } + if(is_use_ahbe){ + // set additional header by ahbe conf + requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); + } + + //DPRN("SHUNDEBUG6PUT %s", string("Date:"+date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2191,6 +2457,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd) bodydata = new BodyData(); // Make request headers +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); @@ -2242,6 +2509,96 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd) string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("PUT", strMD5, ContentType, date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=tpath; + + string payload_hash = s3fs_sha256sum(fd, 0, -1); + if(0 == strlen(payload_hash.c_str())) + payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + //string canonical_headers, signed_headers; + string ContentType; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; + string signed_headers = "host"; + string strMD5; + + // "x-amz-acl" + requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); + canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n"; + signed_headers += ";x-amz-acl"; + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; + signed_headers += ";x-amz-content-sha256"; + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + canonical_headers += "x-amz-date:" + date3 + "\n"; + signed_headers += ";x-amz-date"; + + for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ + string key = (*iter).first; + string value = (*iter).second; + if(0 == strcasecmp(key.c_str(), "Content-Type")){ + ContentType = value; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type:" + value).c_str()); + canonical_headers.insert(0, "content-type:" + value +"\n"); + signed_headers.insert(0, "content-type;"); + }else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ + // not set value, but after set it. + }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ + requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + canonical_headers += key + ":" + value + "\n"; + signed_headers += ";" + key; + }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ + // skip this header, because this header is specified after logic. + }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ + // skip this header, because this header is specified after logic. + }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ + // skip this header, because this header is specified after logic. + } + } + + if(-1 != fd && S3fsCurl::is_content_md5){ + // content-md5 must be before content-type + strMD5 = s3fs_get_content_md5(fd); + // content-md5 must be before content-type + requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-md5: " + strMD5).c_str()); + canonical_headers.insert(0, "content-md5:" + strMD5 +"\n"); + signed_headers.insert(0, "content-md5;"); + } + if(S3fsCurl::is_use_sse){ + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); + canonical_headers += "x-amz-server-side-encryption:AES256\n"; + signed_headers += ";x-amz-server-side-encryption"; + }else if(S3fsCurl::IsSseCustomMode()){ + string md5; + if(!AddSseKeyRequestHead(md5, false)){ + DPRNNN("Failed to insert sse(-c) header."); + } + } + if(S3fsCurl::is_use_rrs){ + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); + canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n"; + signed_headers += ";x-amz-storage-class"; + } + if(is_use_ahbe){ + // set additional header by ahbe conf + requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); + } + //DPRN("SHUNDEBUG5PUT %s", string("Date:"+date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2290,6 +2647,7 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_ requestHeaders = NULL; responseHeaders.clear(); +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); @@ -2312,6 +2670,41 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_ string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("GET", "", "", date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=tpath; + + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ + + payload_hash + "\nx-amz-date:" + date3 + "\n"; + string signed_headers = "host;x-amz-content-sha256;x-amz-date"; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + + //DPRN("SHUNDEBUGxSIGN %s", string("Date:"+date3).c_str()); + if(-1 != start && -1 != size){ + string range = "Range: bytes="; + range += str(start); + range += "-"; + range += str(start + size - 1); + requestHeaders = curl_slist_sort_insert(requestHeaders, range.c_str()); + } + + //DPRN("SHUNDEBUGxSIGN"); + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("GET", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2378,6 +2771,7 @@ int S3fsCurl::CheckBucket(void) responseHeaders.clear(); bodydata = new BodyData(); +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); @@ -2387,6 +2781,37 @@ int S3fsCurl::CheckBucket(void) string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("GET", "", "", date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ + + payload_hash + "\nx-amz-date:" + date3 + "\n"; + string signed_headers = "host;x-amz-content-sha256;x-amz-date"; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + + //DPRN("SHUNDEBUG1SIGN %s", string("Date:"+date3).c_str()); + // step 1 method : PARAM1: 'GET' + // step 2 c..uri : fixed: '/' + // step 3 query string: '' + // step 4 c..headers : PARAM3: 3 + // step 5 s..headers: PARAM5: 3 + // step 6 payload_hash : PARAM4: emptyhash + // step 7 c..request in calcsignature 1+...+6 + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("GET", "", date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif + // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata); @@ -2429,6 +2854,7 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query) responseHeaders.clear(); bodydata = new BodyData(); +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); @@ -2439,6 +2865,32 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query) string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("GET", "", "", date, (resource + "/"))).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=query; + //DPRN("SHUNDEBUG2SIGN %s\t%s", string("CURL:"+ canonical_uri).c_str(), tpath); + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string signed_headers = "host;x-amz-content-sha256;x-amz-date"; + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ + + payload_hash + "\nx-amz-date:" + date3 + "\n"; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + + //DPRN("SHUNDEBUG2SIGN %s", string("Date:"+date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("GET", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2474,6 +2926,7 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); +#ifdef SIGV3 turl += "?uploads"; resource += "?uploads"; url = prepare_url(turl.c_str()); @@ -2531,6 +2984,103 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("POST", "", contype, date, resource)).c_str()); } +#else + string query_string = "uploads="; + turl += "?"+query_string; + resource += "?"+query_string; + url = prepare_url(turl.c_str()); + path = tpath; + requestHeaders = NULL; + responseHeaders.clear(); + bodydata = new BodyData(); + + DPRN("SHUNDEBUG9POST %s", tpath); //string("Date:"+date3).c_str()); + // to be done: what's the payload_hash value? + string payload_hash = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=tpath; + string contype = S3fsCurl::LookupMimeType(string(tpath)); + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; + string signed_headers = "host"; + + // "x-amz-acl" + requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); + canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n"; + signed_headers += ";x-amz-acl"; + + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; + signed_headers += ";x-amz-content-sha256"; + + + requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: "); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length: "); + + requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type: " + contype).c_str()); + canonical_headers.insert(0, "content-type:" + contype +"\n"); + signed_headers.insert(0, "content-type;"); + + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + canonical_headers += "x-amz-date:" + date3 + "\n"; + signed_headers += ";x-amz-date"; + + for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ + string key = (*iter).first; + string value = (*iter).second; + + if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ + // not set value, but after set it. + }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ + requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + canonical_headers += key + ":" + value + "\n"; + signed_headers += ";" + key; + }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ + // skip this header, because this header is specified after logic. + }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ + // skip this header, because this header is specified with "x-amz-...-customer-key-md5". + }else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ + // Only copy mode. + if(!AddSseKeyRequestHead(value, is_copy)){ + DPRNNN("Failed to insert sse(-c) header."); + } + } + } + + // rrs, sse + if(S3fsCurl::is_use_sse){ + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); + canonical_headers += "x-amz-server-side-encryption:AES256\n"; + signed_headers += ";x-amz-server-side-encryption"; + }else if(S3fsCurl::IsSseCustomMode()){ + string md5; + if(!AddSseKeyRequestHead(md5, false)){ + DPRNNN("Failed to insert sse(-c) header."); + } + } + if(S3fsCurl::is_use_rrs){ + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); + canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n"; + signed_headers += ";x-amz-storage-class"; + } + if(is_use_ahbe){ + // set additional header by ahbe conf + requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); + } + + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignatureReal("POST", canonical_uri, query_string, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2598,6 +3148,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id, string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); +#ifdef SIGV3 turl += "?uploadId=" + upload_id; resource += "?uploadId=" + upload_id; url = prepare_url(turl.c_str()); @@ -2617,6 +3168,67 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id, string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("POST", "", "", date, resource)).c_str()); } +#else + string query_string; + query_string = "uploadId=" + upload_id; + turl += "?" + query_string; + resource += "?" + query_string; + url = prepare_url(turl.c_str()); + path = tpath; + requestHeaders = NULL; + responseHeaders.clear(); + bodydata = new BodyData(); + + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + DPRN("SHUNDEBUG9POST %s", tpath); //string("Date:"+date3).c_str()); + + // to be done: what's the payload_hash value? + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + unsigned char * cRequest = (unsigned char *)postContent.c_str(), *sRequest; + unsigned int cRequest_len= postContent.size(); + unsigned int sRequest_len = 0; + char hexsRequest[64]; + unsigned int i; + s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len); + for (i=0;i < sRequest_len;i++) sprintf(hexsRequest+(i*2), "%02x", sRequest[i]); + + + payload_hash.assign(hexsRequest, hexsRequest+sRequest_len*2); + + string canonical_uri = ""; + canonical_uri +=tpath; + string contype = S3fsCurl::LookupMimeType(string(tpath)); + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; + string signed_headers = "host"; + + requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept:"); + requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type: " + contype).c_str()); + canonical_headers.insert(0, "content-type:" + contype +"\n"); + signed_headers.insert(0, "content-type;"); + + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; + signed_headers += ";x-amz-content-sha256"; + + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + canonical_headers += "x-amz-date:" + date3 + "\n"; + signed_headers += ";x-amz-date"; + + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignatureReal("POST", canonical_uri, query_string, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2658,6 +3270,7 @@ int S3fsCurl::MultipartListRequest(string& body) responseHeaders.clear(); bodydata = new BodyData(); +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: "); @@ -2668,6 +3281,29 @@ int S3fsCurl::MultipartListRequest(string& body) string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("GET", "", "", date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ + + payload_hash + "\nx-amz-date:" + date3 + "\n"; + string signed_headers = "host;x-amz-content-sha256;x-amz-date"; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + + //DPRN("SHUNDEBUG4SIGN %s", string("Date:"+date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("GET", "", date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2710,6 +3346,7 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, string& upload_id) requestHeaders = NULL; responseHeaders.clear(); +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); if(!S3fsCurl::IsPublicBucket()){ @@ -2718,6 +3355,30 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, string& upload_id) string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("DELETE", "", "", date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=tpath; + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ + + payload_hash + "\nx-amz-date:" + date3 + "\n"; + string signed_headers = "host;x-amz-content-sha256;x-amz-date"; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + + "/" + date2 + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("DELETE", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); @@ -2764,6 +3425,7 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string& } // make request +#ifdef SIGV3 string urlargs = "?partNumber=" + IntToStr(part_num) + "&uploadId=" + upload_id; string resource; string turl; @@ -2788,6 +3450,55 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string& string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("PUT", "", "", date, resource)).c_str()); } +#else + string request_uri = "partNumber=" + IntToStr(part_num) + "&uploadId=" + upload_id; + string urlargs = "?" + request_uri; + string resource; + string turl; + MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); + + resource += urlargs; + turl += urlargs; + url = prepare_url(turl.c_str()); + path = tpath; + requestHeaders = NULL; + responseHeaders.clear(); + bodydata = new BodyData(); + headdata = new BodyData(); + + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=tpath; + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + string ContentType; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; + string signed_headers = "host"; + + //payload_hash="AWS4-HMAC-SHA256-PAYLOAD"; + payload_hash = s3fs_sha256sum(partdata.fd, partdata.startpos, partdata.size); + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; + signed_headers += ";x-amz-content-sha256"; + + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + canonical_headers += "x-amz-date:" + date3 + "\n"; + signed_headers += ";x-amz-date"; + + DPRN("SHUNDEBUG7PUT %s", string("Date:"+date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignatureReal("PUT", canonical_uri, request_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2861,6 +3572,7 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par headdata = new BodyData(); // Make request headers +#ifdef SIGV3 string date = get_date(); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); @@ -2884,6 +3596,68 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par string("Authorization: AWS " + AWSAccessKeyId + ":" + CalcSignature("PUT", "", ContentType, date, resource)).c_str()); } +#else + string date = get_date(); + string date2 = get_date2(); + string date3 = get_date3(); + string canonical_uri = ""; + canonical_uri +=path; + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + + string ContentType; + requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); + string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; + string signed_headers = "host"; +/* not need + // "x-amz-acl" + requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); + canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n"; + signed_headers += ";x-amz-acl"; +*/ + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-content-sha256:" + payload_hash).c_str()); + canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; + signed_headers += ";x-amz-content-sha256"; + + requestHeaders = curl_slist_sort_insert(requestHeaders, + string("x-amz-date:" + date3).c_str()); + canonical_headers += "x-amz-date:" + date3 + "\n"; + signed_headers += ";x-amz-date"; + + for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ + string key = (*iter).first; + string value = (*iter).second; + if(0 == strcasecmp(key.c_str(), "Content-Type")){ + ContentType = value; + requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + canonical_headers.insert(0, "content-type:" + value + "\n"); + signed_headers.insert(0, "content-type;"); + }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){ + requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n")); + signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key)); + }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source-range")){ + requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n")); + signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key)); + } + // NOTICE: x-amz-acl, x-amz-server-side-encryption is not set! + } + if(S3fsCurl::is_use_rrs){ + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); + canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n"; + signed_headers += ";x-amz-storage-class"; + } + + DPRN("SHUNDEBUG8PUT %s", string("Date:"+date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + requestHeaders = curl_slist_sort_insert( + requestHeaders, + string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 + + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + + CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + } +#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); diff --git a/src/curl.h b/src/curl.h index 75de1b8..1430edf 100644 --- a/src/curl.h +++ b/src/curl.h @@ -244,7 +244,15 @@ class S3fsCurl bool ResetHandle(void); bool RemakeHandle(void); bool ClearInternalData(void); +#ifdef SIGV3 std::string CalcSignature(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource); +#else + std::string CalcSignaturev2(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource); + std::string CalcSignature(std::string method, std::string canonical_uri, std::string date2, std::string cononical_headers, + std::string payload_hash, std::string signed_headers, std::string date3); + std::string CalcSignatureReal(std::string method, std::string canonical_uri, std::string query_string, std::string date2, std::string cononical_headers, + std::string payload_hash, std::string signed_headers, std::string date3); +#endif bool GetUploadId(std::string& upload_id); int GetIAMCredentials(void); diff --git a/src/openssl_auth.cpp b/src/openssl_auth.cpp index 8476854..cafd41b 100644 --- a/src/openssl_auth.cpp +++ b/src/openssl_auth.cpp @@ -32,6 +32,9 @@ #include #include #include +#ifndef SIGV3 +#include +#endif #include #include #include @@ -191,7 +194,11 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){ return false; } +#ifdef SIGV3 HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen); +#else + HMAC(EVP_sha256(), key, keylen, data, datalen, *digest, digestlen); +#endif return true; } @@ -255,6 +262,85 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size) return result; } +#ifndef SIGV3 +//------------------------------------------------------------------- +// Utility Function for SHA256 +//------------------------------------------------------------------- +size_t get_sha256_digest_length(void) +{ + return SHA256_DIGEST_LENGTH; +} + +bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen) +{ + (*digestlen) = EVP_MAX_MD_SIZE * sizeof(unsigned char); + if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){ + return false; + } + + const EVP_MD *md = EVP_get_digestbyname("sha256"); + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + EVP_DigestInit_ex(mdctx, md, NULL); + EVP_DigestUpdate(mdctx, data, datalen); + EVP_DigestFinal_ex(mdctx, *digest, digestlen); + EVP_MD_CTX_destroy(mdctx); + + return true; +} + +unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) +{ + + const EVP_MD *md = EVP_get_digestbyname("sha256"); + EVP_MD_CTX *sha256ctx = EVP_MD_CTX_create(); + EVP_DigestInit_ex(sha256ctx, md, NULL); + + char buf[512]; + ssize_t bytes; + unsigned char* result; + + if(-1 == size) { + struct stat st; + if(-1 == fstat(fd, &st)){ + return NULL; + } + size = static_cast(st.st_size); + } + + // seek to top of file. + if(-1 == lseek(fd, start, SEEK_SET)){ + return NULL; + } + + memset(buf, 0, 512); + for(ssize_t total = 0; total < size; total += bytes){ + bytes = 512 < (size - total) ? 512 : (size - total); + bytes = read(fd, buf, bytes); + if(0 == bytes){ + // end of file + break; + }else if(-1 == bytes){ + // error + DPRNNN("file read error(%d)", errno); + return NULL; + } + EVP_DigestUpdate(sha256ctx, buf, bytes); + memset(buf, 0, 512); + } + if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){ + return NULL; + } + EVP_DigestFinal_ex(sha256ctx, result, NULL); + EVP_MD_CTX_destroy(sha256ctx); + + if(-1 == lseek(fd, start, SEEK_SET)){ + free(result); + return NULL; + } + return result; +} +#endif + /* * Local variables: * tab-width: 4 diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 86a0fce..82d3c1d 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -88,6 +88,9 @@ std::string program_name; std::string service_path = "/"; std::string host = "http://s3.amazonaws.com"; std::string bucket = ""; +#ifndef SIGV3 +std::string endpoint = "us-east-1"; +#endif //------------------------------------------------------------------- // Static valiables @@ -3759,6 +3762,12 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar } return 0; } +#ifndef SIGV3 + if(0 == STR2NCMP(arg, "endpoint=")){ + endpoint = strchr(arg, '=') + sizeof(char); + return 0; + } +#endif if(0 == strcmp(arg, "use_path_request_style")){ pathrequeststyle = true; return 0; diff --git a/src/s3fs_auth.h b/src/s3fs_auth.h index ad1f060..fb3fad1 100644 --- a/src/s3fs_auth.h +++ b/src/s3fs_auth.h @@ -29,6 +29,9 @@ char* s3fs_base64(unsigned char* input, size_t length); std::string s3fs_get_content_md5(int fd); std::string s3fs_md5sum(int fd, off_t start, ssize_t size); +#ifndef SIGV3 +std::string s3fs_sha256sum(int fd, off_t start, ssize_t size); +#endif // // in xxxxxx_auth.cpp @@ -41,6 +44,11 @@ bool s3fs_destroy_crypt_mutex(void); bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen); size_t get_md5_digest_length(void); unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size); +#ifndef SIGV3 +bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen); +size_t get_sha256_digest_length(void); +unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size); +#endif #endif // S3FS_AUTH_H_ diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index 88a9436..43b889e 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -975,6 +975,11 @@ void show_help (void) "\n" " url (default=\"http://s3.amazonaws.com\")\n" " - sets the url to use to access amazon s3\n" +#ifndef SIGV3 + "\n" + " endpoint (default=\"us-east-1\")\n" + " - sets the endpoint to use\n" +#endif "\n" " nomultipart (disable multipart uploads)\n" "\n" diff --git a/src/string_util.cpp b/src/string_util.cpp index 9bc1a3a..c3efe1f 100644 --- a/src/string_util.cpp +++ b/src/string_util.cpp @@ -125,6 +125,12 @@ string urlEncode(const string &s) for (unsigned i = 0; i < s.length(); ++i) { if (s[i] == '/') { // Note- special case for fuse paths... result += s[i]; +#ifndef SIGV3 + }else if (s[i] == '=') { // Note- special case for s3... + result += s[i]; + }else if (s[i] == '&') { // Note- special case for s3... + result += s[i]; +#endif } else if (isalnum(s[i])) { result += s[i]; } else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') { @@ -143,6 +149,39 @@ string urlEncode(const string &s) return result; } +#ifndef SIGV3 +/** + * urlEncode a fuse path, + * taking into special consideration "/", + * otherwise regular urlEncode. + */ +string urlEncode2(const string &s) +{ + string result; + for (unsigned i = 0; i < s.length(); ++i) { + if (s[i] == '=') { // Note- special case for fuse paths... + result += s[i]; + }else if (s[i] == '&') { // Note- special case for s3... + result += s[i]; + } else if (isalnum(s[i])) { + result += s[i]; + } else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') { + result += s[i]; + } else if (s[i] == ' ') { + result += '%'; + result += '2'; + result += '0'; + } else { + result += "%"; + result += hexAlphabet[static_cast(s[i]) / 16]; + result += hexAlphabet[static_cast(s[i]) % 16]; + } + } + + return result; +} +#endif + // // ex. target="http://......?keyword=value&..." // @@ -181,6 +220,25 @@ string get_date() return buf; } +#ifndef SIGV3 +string get_date2() +{ + char buf[100]; + time_t t = time(NULL); + strftime(buf, sizeof(buf), "%Y%m%d", gmtime(&t)); + return buf; +} + +string get_date3() +{ + char buf[100]; + time_t t = time(NULL); + strftime(buf, sizeof(buf), "%Y%m%dT%H%M%SZ", gmtime(&t)); + return buf; +} + +#endif + /* * Local variables: * tab-width: 4 diff --git a/src/string_util.h b/src/string_util.h index 13f7447..cad1239 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -47,7 +47,14 @@ std::string trim(const std::string &s, const std::string &t = SPACES); std::string lower(std::string s); std::string IntToStr(int); std::string get_date(); +#ifndef SIGV3 +std::string get_date2(); +std::string get_date3(); +#endif std::string urlEncode(const std::string &s); +#ifndef SIGV3 +std::string urlEncode2(const std::string &s); +#endif bool get_keyword_value(std::string& target, const char* keyword, std::string& value); #endif // S3FS_STRING_UTIL_H_ From 0d2f3e2dc4d61536ae91204f46bc0b4eb57e926e Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Sat, 24 Jan 2015 16:36:30 +0000 Subject: [PATCH 2/4] Fixed bugs, segfault and signature error at listing. --- src/curl.cpp | 66 +++++++++++++++++++++++++++++----------------------- src/s3fs.cpp | 27 ++++++++++++--------- 2 files changed, 53 insertions(+), 40 deletions(-) diff --git a/src/curl.cpp b/src/curl.cpp index 14b51d6..3fde319 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -1839,8 +1839,7 @@ string S3fsCurl::CalcSignaturev2(string method, string strMD5, string content_ty return Signature; } -string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string query_string , string date2, - string canonical_headers, string payload_hash, string signed_headers, string date3) +string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string query_string, string date2, string canonical_headers, string payload_hash, string signed_headers, string date3) { string Signature, StringCQ, StringToSign; string uriencode; @@ -1854,54 +1853,63 @@ string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string q } uriencode = urlEncode(canonical_uri); - StringCQ = method + "\n"; + StringCQ = method + "\n"; if(0 == strcmp(method.c_str(),"HEAD") || 0 == strcmp(method.c_str(),"PUT") || 0 == strcmp(method.c_str(),"DELETE")){ StringCQ += uriencode + "\n" + query_string + "\n"; }else if (0 == strcmp(method.c_str(), "GET") && 0 == strcmp(uriencode.c_str(), "")) { StringCQ +="/\n\n"; - }else if (0 == strcmp(method.c_str(), "GET") && 0 == strncmp(uriencode.c_str(), "/",1)) { + }else if (0 == strcmp(method.c_str(), "GET") && 0 == strncmp(uriencode.c_str(), "/", 1)) { StringCQ += uriencode +"\n\n"; - }else if (0 == strcmp(method.c_str(), "GET") && 0 != strncmp(uriencode.c_str(), "/",1)) { + }else if (0 == strcmp(method.c_str(), "GET") && 0 != strncmp(uriencode.c_str(), "/", 1)) { StringCQ += "/\n" + urlEncode2(canonical_uri) +"\n"; }else if (0 == strcmp(method.c_str(), "POST")) { - StringCQ += uriencode +"\n" + query_string +"\n"; + StringCQ += uriencode +"\n" + query_string +"\n"; } StringCQ += canonical_headers + "\n"; StringCQ += signed_headers + "\n"; StringCQ += payload_hash; - unsigned char * cRequest = (unsigned char *)StringCQ.c_str(); - unsigned int cRequest_len= StringCQ.size(); + + char kSecret[128]; + unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest = NULL; + unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0; + char hexsRequest[64 + 1]; + int kSecret_len = snprintf(kSecret, sizeof(kSecret), "AWS4%s", S3fsCurl::AWSSecretAccessKey.c_str()); + unsigned int i; + + s3fs_HMAC(kSecret, kSecret_len, reinterpret_cast(date2.data()), date2.size(), &kDate, &kDate_len); + s3fs_HMAC(kDate, kDate_len, reinterpret_cast(endpoint.c_str()), endpoint.size(), &kRegion, &kRegion_len); + s3fs_HMAC(kRegion, kRegion_len, reinterpret_cast("s3"), sizeof("s3") - 1, &kService, &kService_len); + s3fs_HMAC(kService, kService_len, reinterpret_cast("aws4_request"), sizeof("aws4_request") - 1, &kSigning, &kSigning_len); + free(kDate); + free(kRegion); + free(kService); + + const unsigned char* cRequest = reinterpret_cast(StringCQ.c_str()); + unsigned int cRequest_len = StringCQ.size(); // DPRN("SHUNDEBUGXXXPUT: %s", cRequest); - char kSecret[128]; - unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest = NULL; - unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0; - char hexsRequest[64]; - int kSecret_len = snprintf(kSecret, sizeof(kSecret), "AWS4%s", S3fsCurl::AWSSecretAccessKey.c_str()); - unsigned int i; - - s3fs_HMAC(kSecret, kSecret_len, (unsigned char*)date2.data(), date2.size(), &kDate, &kDate_len); - s3fs_HMAC(kDate, kDate_len, (unsigned char *)endpoint.c_str(), endpoint.size(), &kRegion, &kRegion_len); - s3fs_HMAC(kRegion, kRegion_len, (unsigned char *)"s3", sizeof("s3")-1, &kService, &kService_len); - s3fs_HMAC(kService, kService_len, (unsigned char *)"aws4_request", sizeof("aws4_request")-1, &kSigning, &kSigning_len); - s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len); - //for (i=0;i < sRequest_len;i++) printf("%02x", sRequest[i]); - - for (i=0;i < sRequest_len;i++) sprintf(hexsRequest+(i*2), "%02x", sRequest[i]); + for(i = 0; i < sRequest_len; i++){ + sprintf(&hexsRequest[i * 2], "%02x", sRequest[i]); + } + free(sRequest); StringToSign = "AWS4-HMAC-SHA256\n"; StringToSign += date3+"\n"; StringToSign += date2+"/" + endpoint + "/s3/aws4_request\n"; StringToSign += hexsRequest; + const unsigned char* cscope = reinterpret_cast(StringToSign.c_str()); + unsigned int cscope_len = StringToSign.size(); + unsigned char* md = NULL; + unsigned int md_len = 0; + char hexSig[64 + 1]; - unsigned char* cscope = (unsigned char*)StringToSign.c_str(); - unsigned int cscope_len = StringToSign.size(); - unsigned char* md = NULL; - unsigned int md_len = 0; - char hexSig[64]; s3fs_HMAC(kSigning, kSigning_len, cscope, cscope_len, &md, &md_len); - for (i=0; i < md_len; i++) sprintf(hexSig+(i*2), "%02x", md[i]); + for(i = 0; i < md_len; i++){ + sprintf(&hexSig[i * 2], "%02x", md[i]); + } + free(kSigning); + free(md); Signature = hexSig; diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 82d3c1d..e69f5fd 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -2264,7 +2264,9 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, { int result; string s3_realpath; - string query; + string query_delimiter;; + string query_prefix;; + string query_maxkey;; string next_marker = ""; bool truncated = true; S3fsCurl s3fscurl; @@ -2274,31 +2276,34 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter, FPRNN("[path=%s]", path); if(delimiter && 0 < strlen(delimiter)){ - query += "delimiter="; - query += delimiter; - query += "&"; + query_delimiter += "delimiter="; + query_delimiter += delimiter; + query_delimiter += "&"; } - query += "prefix="; + query_prefix += "&prefix="; s3_realpath = get_realpath(path); if(0 == s3_realpath.length() || '/' != s3_realpath[s3_realpath.length() - 1]){ // last word must be "/" - query += urlEncode(s3_realpath.substr(1) + "/"); + query_prefix += urlEncode(s3_realpath.substr(1) + "/"); }else{ - query += urlEncode(s3_realpath.substr(1)); + query_prefix += urlEncode(s3_realpath.substr(1)); } if (check_content_only){ - query += "&max-keys=1"; + query_maxkey += "max-keys=1"; }else{ - query += "&max-keys=1000"; + query_maxkey += "max-keys=1000"; } while(truncated){ - string each_query = query; + string each_query = query_delimiter; if(next_marker != ""){ - each_query += "&marker=" + urlEncode(next_marker); + each_query += "marker=" + urlEncode(next_marker) + "&"; next_marker = ""; } + each_query += query_maxkey; + each_query += query_prefix; + // request if(0 != (result = s3fscurl.ListBucketRequest(path, each_query.c_str()))){ DPRN("ListBucketRequest returns with error."); From 4f953f9bd78c81afd825dab49a81ad2a68db0be4 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Wed, 28 Jan 2015 17:13:11 +0000 Subject: [PATCH 3/4] Clean codes for signature v4 and added new sigv2 option --- doc/man/s3fs.1 | 6 + src/cache.cpp | 29 +- src/common.h | 2 - src/common_auth.cpp | 2 - src/curl.cpp | 1476 ++++++++++++++++-------------------------- src/curl.h | 17 +- src/openssl_auth.cpp | 43 +- src/s3fs.cpp | 8 +- src/s3fs_auth.h | 5 +- src/s3fs_util.cpp | 5 +- src/string_util.cpp | 28 +- src/string_util.h | 11 +- 12 files changed, 631 insertions(+), 1001 deletions(-) diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index b3169df..438bf0a 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -149,6 +149,12 @@ This option is lated to fd_page_size option and affects it. \fB\-o\fR url (default="http://s3.amazonaws.com") sets the url to use to access Amazon S3. If you want to use HTTPS, then you can set url=https://s3.amazonaws.com .TP +\fB\-o\fR endpoint (default="us-east-1") +sets the endpoint to use. +.TP +\fB\-o\fR sigv2 (default is signature version 4) +sets signing AWS requests by sing Signature Version 2. +.TP \fB\-o\fR nomultipart - disable multipart uploads .TP \fB\-o\fR enable_content_md5 ( default is disable ) diff --git a/src/cache.cpp b/src/cache.cpp index f9957c1..f3f63c4 100644 --- a/src/cache.cpp +++ b/src/cache.cpp @@ -35,6 +35,7 @@ #include "cache.h" #include "s3fs.h" #include "s3fs_util.h" +#include "string_util.h" using namespace std; @@ -269,24 +270,18 @@ bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir) ent->meta.clear(); //copy only some keys for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string tag = (*iter).first; - string value = (*iter).second; - if(tag == "Content-Type"){ - ent->meta[tag] = value; - }else if(tag == "Content-Length"){ - ent->meta[tag] = value; - }else if(tag == "ETag"){ - ent->meta[tag] = value; - }else if(tag == "Last-Modified"){ - ent->meta[tag] = value; + string tag = lower(iter->first); + string value = iter->second; + if(tag == "content-type"){ + ent->meta[iter->first] = value; + }else if(tag == "content-length"){ + ent->meta[iter->first] = value; + }else if(tag == "etag"){ + ent->meta[iter->first] = value; + }else if(tag == "last-modified"){ + ent->meta[iter->first] = value; }else if(tag.substr(0, 5) == "x-amz"){ - ent->meta[tag] = value; - }else{ - // Check for upper case - transform(tag.begin(), tag.end(), tag.begin(), static_cast(std::tolower)); - if(tag.substr(0, 5) == "x-amz"){ - ent->meta[tag] = value; - } + ent->meta[tag] = value; // key is lower case for "x-amz" } } // add diff --git a/src/common.h b/src/common.h index cb7a4ea..1c0d39d 100644 --- a/src/common.h +++ b/src/common.h @@ -94,9 +94,7 @@ extern std::string service_path; extern std::string host; extern std::string bucket; extern std::string mount_prefix; -#ifndef SIGV3 extern std::string endpoint; -#endif #endif // S3FS_COMMON_H_ diff --git a/src/common_auth.cpp b/src/common_auth.cpp index 14b7e0f..ab8d7ee 100644 --- a/src/common_auth.cpp +++ b/src/common_auth.cpp @@ -102,7 +102,6 @@ string s3fs_md5sum(int fd, off_t start, ssize_t size) return string(md5); } -#ifndef SIGV3 string s3fs_sha256sum(int fd, off_t start, ssize_t size) { size_t digestlen = get_sha256_digest_length(); @@ -123,7 +122,6 @@ string s3fs_sha256sum(int fd, off_t start, ssize_t size) return string(sha256); } -#endif /* * Local variables: diff --git a/src/curl.cpp b/src/curl.cpp index 3fde319..2fcec94 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -227,6 +227,7 @@ string S3fsCurl::curl_ca_bundle; mimes_t S3fsCurl::mimeTypes; int S3fsCurl::max_parallel_cnt = 5; // default off_t S3fsCurl::multipart_size = MULTIPART_SIZE; // default +bool S3fsCurl::is_sigv4 = true; // default //------------------------------------------------------------------- // Class methods for S3fsCurl @@ -1739,60 +1740,10 @@ int S3fsCurl::RequestPerform(void) // // @param method e.g., "GET" // @param content_type e.g., "application/x-directory" -// @param date e.g., get_date() +// @param date e.g., get_date_rfc850() // @param resource e.g., "/pub" // -#ifdef SIGV3 -string S3fsCurl::CalcSignature(string method, string strMD5, string content_type, string date, string resource) -{ - string Signature; - string StringToSign; - - if(0 < S3fsCurl::IAM_role.size()){ - if(!S3fsCurl::CheckIAMCredentialUpdate()){ - DPRN("Something error occurred in checking IAM credential."); - return Signature; // returns empty string, then it occures error. - } - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str()); - } - - StringToSign += method + "\n"; - StringToSign += strMD5 + "\n"; // md5 - StringToSign += content_type + "\n"; - StringToSign += date + "\n"; - for(curl_slist* headers = requestHeaders; headers; headers = headers->next){ - if(0 == strncmp(headers->data, "x-amz", 5)){ - StringToSign += headers->data; - StringToSign += "\n"; - } - } - StringToSign += resource; - - const void* key = S3fsCurl::AWSSecretAccessKey.data(); - int key_len = S3fsCurl::AWSSecretAccessKey.size(); - const unsigned char* sdata = reinterpret_cast(StringToSign.data()); - int sdata_len = StringToSign.size(); - unsigned char* md = NULL; - unsigned int md_len = 0;; - - s3fs_HMAC(key, key_len, sdata, sdata_len, &md, &md_len); - - char* base64; - if(NULL == (base64 = s3fs_base64(md, md_len))){ - free(md); - return string(""); // ENOMEM - } - free(md); - - Signature = base64; - free(base64); - - return Signature; -} - -#else -// NOT SIGV3 -string S3fsCurl::CalcSignaturev2(string method, string strMD5, string content_type, string date, string resource) +string S3fsCurl::CalcSignatureV2(string method, string strMD5, string content_type, string date, string resource) { string Signature; string StringToSign; @@ -1802,19 +1753,14 @@ string S3fsCurl::CalcSignaturev2(string method, string strMD5, string content_ty DPRN("Something error occurred in checking IAM credential."); return Signature; // returns empty string, then it occures error. } - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::AWSAccessToken.c_str()); } StringToSign += method + "\n"; StringToSign += strMD5 + "\n"; // md5 StringToSign += content_type + "\n"; StringToSign += date + "\n"; - for(curl_slist* headers = requestHeaders; headers; headers = headers->next){ - if(0 == strncmp(headers->data, "x-amz", 5)){ - StringToSign += headers->data; - StringToSign += "\n"; - } - } + StringToSign += get_canonical_headers(requestHeaders, true); StringToSign += resource; const void* key = S3fsCurl::AWSSecretAccessKey.data(); @@ -1839,7 +1785,7 @@ string S3fsCurl::CalcSignaturev2(string method, string strMD5, string content_ty return Signature; } -string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string query_string, string date2, string canonical_headers, string payload_hash, string signed_headers, string date3) +string S3fsCurl::CalcSignature(string method, string canonical_uri, string query_string, string strdate, string payload_hash, string date8601) { string Signature, StringCQ, StringToSign; string uriencode; @@ -1849,7 +1795,7 @@ string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string q DPRN("Something error occurred in checking IAM credential."); return Signature; // returns empty string, then it occures error. } - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", S3fsCurl::AWSAccessToken.c_str()); } uriencode = urlEncode(canonical_uri); @@ -1865,8 +1811,8 @@ string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string q }else if (0 == strcmp(method.c_str(), "POST")) { StringCQ += uriencode +"\n" + query_string +"\n"; } - StringCQ += canonical_headers + "\n"; - StringCQ += signed_headers + "\n"; + StringCQ += get_canonical_headers(requestHeaders) + "\n"; + StringCQ += get_sorted_header_keys(requestHeaders) + "\n"; StringCQ += payload_hash; char kSecret[128]; @@ -1874,39 +1820,38 @@ string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string q unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0; char hexsRequest[64 + 1]; int kSecret_len = snprintf(kSecret, sizeof(kSecret), "AWS4%s", S3fsCurl::AWSSecretAccessKey.c_str()); - unsigned int i; + unsigned int cnt; - s3fs_HMAC(kSecret, kSecret_len, reinterpret_cast(date2.data()), date2.size(), &kDate, &kDate_len); - s3fs_HMAC(kDate, kDate_len, reinterpret_cast(endpoint.c_str()), endpoint.size(), &kRegion, &kRegion_len); - s3fs_HMAC(kRegion, kRegion_len, reinterpret_cast("s3"), sizeof("s3") - 1, &kService, &kService_len); - s3fs_HMAC(kService, kService_len, reinterpret_cast("aws4_request"), sizeof("aws4_request") - 1, &kSigning, &kSigning_len); + s3fs_HMAC256(kSecret, kSecret_len, reinterpret_cast(strdate.data()), strdate.size(), &kDate, &kDate_len); + s3fs_HMAC256(kDate, kDate_len, reinterpret_cast(endpoint.c_str()), endpoint.size(), &kRegion, &kRegion_len); + s3fs_HMAC256(kRegion, kRegion_len, reinterpret_cast("s3"), sizeof("s3") - 1, &kService, &kService_len); + s3fs_HMAC256(kService, kService_len, reinterpret_cast("aws4_request"), sizeof("aws4_request") - 1, &kSigning, &kSigning_len); free(kDate); free(kRegion); free(kService); - const unsigned char* cRequest = reinterpret_cast(StringCQ.c_str()); - unsigned int cRequest_len = StringCQ.size(); -// DPRN("SHUNDEBUGXXXPUT: %s", cRequest); + const unsigned char* cRequest = reinterpret_cast(StringCQ.c_str()); + unsigned int cRequest_len = StringCQ.size(); s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len); - for(i = 0; i < sRequest_len; i++){ - sprintf(&hexsRequest[i * 2], "%02x", sRequest[i]); + for(cnt = 0; cnt < sRequest_len; cnt++){ + sprintf(&hexsRequest[cnt * 2], "%02x", sRequest[cnt]); } free(sRequest); - StringToSign = "AWS4-HMAC-SHA256\n"; - StringToSign += date3+"\n"; - StringToSign += date2+"/" + endpoint + "/s3/aws4_request\n"; + StringToSign = "AWS4-HMAC-SHA256\n"; + StringToSign += date8601 + "\n"; + StringToSign += strdate + "/" + endpoint + "/s3/aws4_request\n"; StringToSign += hexsRequest; - const unsigned char* cscope = reinterpret_cast(StringToSign.c_str()); - unsigned int cscope_len = StringToSign.size(); - unsigned char* md = NULL; - unsigned int md_len = 0; - char hexSig[64 + 1]; + const unsigned char* cscope = reinterpret_cast(StringToSign.c_str()); + unsigned int cscope_len = StringToSign.size(); + unsigned char* md = NULL; + unsigned int md_len = 0; + char hexSig[64 + 1]; - s3fs_HMAC(kSigning, kSigning_len, cscope, cscope_len, &md, &md_len); - for(i = 0; i < md_len; i++){ - sprintf(&hexSig[i * 2], "%02x", md[i]); + s3fs_HMAC256(kSigning, kSigning_len, cscope, cscope_len, &md, &md_len); + for(cnt = 0; cnt < md_len; cnt++){ + sprintf(&hexSig[cnt * 2], "%02x", md[cnt]); } free(kSigning); free(md); @@ -1916,13 +1861,6 @@ string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string q return Signature; } -string S3fsCurl::CalcSignature(string method, string canonical_uri, string date2, - string canonical_headers, string payload_hash, string signed_headers, string date3) -{ - return CalcSignatureReal(method, canonical_uri, "", date2, canonical_headers, payload_hash, signed_headers, date3); -} -#endif - // XML in BodyData has UploadId, Parse XML body for UploadId bool S3fsCurl::GetUploadId(string& upload_id) { @@ -1986,40 +1924,32 @@ int S3fsCurl::DeleteRequest(const char* tpath) requestHeaders = NULL; responseHeaders.clear(); -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("DELETE", "", "", date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=tpath; - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ - + payload_hash + "\nx-amz-date:" + date3 + "\n"; - string signed_headers = "host;x-amz-content-sha256;x-amz-date"; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("DELETE", "", "", date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + - "/" + date2 + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("DELETE", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("DELETE", path, "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); @@ -2083,13 +2013,13 @@ bool S3fsCurl::AddSseKeyRequestHead(string& md5, bool is_copy) string sseckey; if(S3fsCurl::GetSseKey(md5, sseckey)){ if(is_copy){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm:AES256"); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-copy-source-server-side-encryption-customer-key:" + sseckey).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-copy-source-server-side-encryption-customer-key-md5:" + md5).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm", "AES256"); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key", sseckey.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key-md5", md5.c_str()); }else{ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-algorithm:AES256"); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-server-side-encryption-customer-key:" + sseckey).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-server-side-encryption-customer-key-md5:" + md5).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-algorithm", "AES256"); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key", sseckey.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key-md5", md5.c_str()); } } return true; @@ -2125,11 +2055,6 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* responseHeaders.clear(); // requestHeaders -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); - if(0 <= ssekey_pos && S3fsCurl::IsSseCustomMode()){ string md5; if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseKeyRequestHead(md5, false)){ @@ -2138,37 +2063,33 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* } b_ssekey_pos = ssekey_pos; - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("HEAD", "", "", date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=tpath; - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ - + payload_hash + "\nx-amz-date:" + date3 + "\n"; - string signed_headers = "host;x-amz-content-sha256;x-amz-date"; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL); - //DPRN("SHUNDEBUG3SIGN %s", string("Date:"+date3).c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("HEAD", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("HEAD", "", "", date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } + + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("HEAD", path, "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_NOBODY, true); // HEAD @@ -2222,24 +2143,18 @@ int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta) // fixme: clean this up. meta.clear(); for(headers_t::iterator iter = responseHeaders.begin(); iter != responseHeaders.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - if(0 == strcasecmp(key.c_str(), "Content-Type")){ - meta[key] = value; - }else if(0 == strcasecmp(key.c_str(), "Content-Length")){ - meta[key] = value; - }else if(0 == strcasecmp(key.c_str(), "ETag")){ - meta[key] = value; - }else if(0 == strcasecmp(key.c_str(), "Last-Modified")){ - meta[key] = value; - }else if(0 == strcasecmp(key.substr(0, 5).c_str(), "x-amz")){ - meta[key] = value; - }else{ - // Check for upper case - transform(key.begin(), key.end(), key.begin(), static_cast(std::tolower)); - if(key.substr(0, 5) == "x-amz"){ - meta[key] = value; - } + string key = lower(iter->first); + string value = iter->second; + if(key == "content-type"){ + meta[iter->first] = value; + }else if(key == "content-length"){ + meta[iter->first] = value; + }else if(key == "etag"){ + meta[iter->first] = value; + }else if(key == "last-modified"){ + meta[iter->first] = value; + }else if(key.substr(0, 5) == "x-amz"){ + meta[key] = value; // key is lower case for "x-amz" } } return 0; @@ -2266,28 +2181,24 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) bodydata = new BodyData(); // Make request headers -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - string ContentType; for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - if(0 == strcasecmp(key.c_str(), "Content-Type")){ + string key = lower(iter->first); + string value = iter->second; + if(key == "content-type"){ ContentType = value; - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - }else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); + }else if(key.substr(0, 9) == "x-amz-acl"){ // not set value, but after set it. - }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ + }else if(key.substr(0, 10) == "x-amz-meta"){ + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); + }else if(key == "x-amz-copy-source"){ + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); + }else if(key == "x-amz-server-side-encryption"){ // skip this header, because this header is specified after logic. - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ + }else if(key == "x-amz-server-side-encryption-customer-algorithm"){ // skip this header, because this header is specified with "x-amz-...-customer-key-md5". - }else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ + }else if(is_copy && key == "x-amz-server-side-encryption-customer-key-md5"){ // Only copy mode. if(!AddSseKeyRequestHead(value, is_copy)){ DPRNNN("Failed to insert sse(-c) header."); @@ -2296,12 +2207,12 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) } // "x-amz-acl", rrs, sse - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.c_str()); if(S3fsCurl::is_use_rrs){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "REDUCED_REDUNDANCY"); } if(S3fsCurl::is_use_sse){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256"); }else if(S3fsCurl::IsSseCustomMode()){ string md5; if(!AddSseKeyRequestHead(md5, false)){ @@ -2312,100 +2223,34 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy) // set additional header by ahbe conf requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); } - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("PUT", "", ContentType, date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=tpath; - //string payload_hash = s3fs_sha256sum(fd, 0, -1); - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string ContentType; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; - string signed_headers = "host"; - // "x-amz-acl" - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); - canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n"; - signed_headers += ";x-amz-acl"; - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; - signed_headers += ";x-amz-content-sha256"; + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); - canonical_headers += "x-amz-date:" + date3 + "\n"; - signed_headers += ";x-amz-date"; + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("PUT", "", ContentType, date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - if(0 == strcasecmp(key.c_str(), "Content-Type")){ - ContentType = value; - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - canonical_headers.insert(0, "content-type:" + value + "\n"); - signed_headers.insert(0, "content-type;"); - }else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ - // not set value, but after set it. - }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - canonical_headers += key + ":" + value + "\n"; - signed_headers += ";" + key; - }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n")); - signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key)); - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ - // skip this header, because this header is specified after logic. - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ - // skip this header, because this header is specified with "x-amz-...-customer-key-md5". - }else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ - // Only copy mode. - if(!AddSseKeyRequestHead(value, is_copy)){ - DPRNNN("Failed to insert sse(-c) header."); - } + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("PUT", path, "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); } } - // rrs, sse - if(S3fsCurl::is_use_sse){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); - canonical_headers += "x-amz-server-side-encryption:AES256\n"; - signed_headers += ";x-amz-server-side-encryption"; - }else if(S3fsCurl::IsSseCustomMode()){ - string md5; - if(!AddSseKeyRequestHead(md5, false)){ - DPRNNN("Failed to insert sse(-c) header."); - } - } - if(S3fsCurl::is_use_rrs){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); - canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n"; - signed_headers += ";x-amz-storage-class"; - } - if(is_use_ahbe){ - // set additional header by ahbe conf - requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); - } - - //DPRN("SHUNDEBUG6PUT %s", string("Date:"+date3).c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); - } -#endif - // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true); // HTTP PUT @@ -2465,42 +2310,38 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd) bodydata = new BodyData(); // Make request headers -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - string strMD5; if(-1 != fd && S3fsCurl::is_content_md5){ strMD5 = s3fs_get_content_md5(fd); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Content-MD5: " + strMD5).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", strMD5.c_str()); } string ContentType; for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - if(0 == strcasecmp(key.c_str(), "Content-Type")){ + string key = lower(iter->first); + string value = iter->second; + if(key == "content-type"){ ContentType = value; - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - }else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); + }else if(key.substr(0, 9) == "x-amz-acl"){ // not set value, but after set it. - }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ + }else if(key.substr(0, 10) == "x-amz-meta"){ + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); + }else if(key == "x-amz-server-side-encryption"){ // skip this header, because this header is specified after logic. - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ + }else if(key == "x-amz-server-side-encryption-customer-algorithm"){ // skip this header, because this header is specified after logic. - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ + }else if(key == "x-amz-server-side-encryption-customer-key-md5"){ // skip this header, because this header is specified after logic. } } // "x-amz-acl", rrs, sse - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.c_str()); if(S3fsCurl::is_use_rrs){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "REDUCED_REDUNDANCY"); } if(S3fsCurl::is_use_sse){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256"); }else if(S3fsCurl::IsSseCustomMode()){ string md5; if(!AddSseKeyRequestHead(md5, false)){ @@ -2511,103 +2352,39 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd) // set additional header by ahbe conf requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); } - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("PUT", strMD5, ContentType, date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=tpath; - string payload_hash = s3fs_sha256sum(fd, 0, -1); - if(0 == strlen(payload_hash.c_str())) - payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); - //string canonical_headers, signed_headers; - string ContentType; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; - string signed_headers = "host"; - string strMD5; + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("PUT", strMD5, ContentType, date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - // "x-amz-acl" - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); - canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n"; - signed_headers += ";x-amz-acl"; - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; - signed_headers += ";x-amz-content-sha256"; - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); - canonical_headers += "x-amz-date:" + date3 + "\n"; - signed_headers += ";x-amz-date"; + }else{ + string payload_hash = s3fs_sha256sum(fd, 0, -1); + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); - for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - if(0 == strcasecmp(key.c_str(), "Content-Type")){ - ContentType = value; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type:" + value).c_str()); - canonical_headers.insert(0, "content-type:" + value +"\n"); - signed_headers.insert(0, "content-type;"); - }else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ - // not set value, but after set it. - }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - canonical_headers += key + ":" + value + "\n"; - signed_headers += ";" + key; - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ - // skip this header, because this header is specified after logic. - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ - // skip this header, because this header is specified after logic. - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ - // skip this header, because this header is specified after logic. + if(0 == payload_hash.length()){ + payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + } + + //string canonical_headers, signed_headers; + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("PUT", path, "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); } } - if(-1 != fd && S3fsCurl::is_content_md5){ - // content-md5 must be before content-type - strMD5 = s3fs_get_content_md5(fd); - // content-md5 must be before content-type - requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-md5: " + strMD5).c_str()); - canonical_headers.insert(0, "content-md5:" + strMD5 +"\n"); - signed_headers.insert(0, "content-md5;"); - } - if(S3fsCurl::is_use_sse){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); - canonical_headers += "x-amz-server-side-encryption:AES256\n"; - signed_headers += ";x-amz-server-side-encryption"; - }else if(S3fsCurl::IsSseCustomMode()){ - string md5; - if(!AddSseKeyRequestHead(md5, false)){ - DPRNNN("Failed to insert sse(-c) header."); - } - } - if(S3fsCurl::is_use_rrs){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); - canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n"; - signed_headers += ";x-amz-storage-class"; - } - if(is_use_ahbe){ - // set additional header by ahbe conf - requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); - } - //DPRN("SHUNDEBUG5PUT %s", string("Date:"+date3).c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); - } -#endif - // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true); // HTTP PUT @@ -2655,16 +2432,12 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_ requestHeaders = NULL; responseHeaders.clear(); -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); if(-1 != start && -1 != size){ - string range = "Range: bytes="; + string range = "bytes="; range += str(start); range += "-"; range += str(start + size - 1); - requestHeaders = curl_slist_sort_insert(requestHeaders, range.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Range", range.c_str()); } if(0 < ssekeymd5.length()){ if(!AddSseKeyRequestHead(ssekeymd5, false)){ @@ -2672,47 +2445,33 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_ } } - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("GET", "", "", date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=tpath; + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL); - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ - + payload_hash + "\nx-amz-date:" + date3 + "\n"; - string signed_headers = "host;x-amz-content-sha256;x-amz-date"; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("GET", "", "", date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - //DPRN("SHUNDEBUGxSIGN %s", string("Date:"+date3).c_str()); - if(-1 != start && -1 != size){ - string range = "Range: bytes="; - range += str(start); - range += "-"; - range += str(start + size - 1); - requestHeaders = curl_slist_sort_insert(requestHeaders, range.c_str()); - } + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); - //DPRN("SHUNDEBUGxSIGN"); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("GET", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("GET", path, "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2779,46 +2538,32 @@ int S3fsCurl::CheckBucket(void) responseHeaders.clear(); bodydata = new BodyData(); -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("GET", "", "", date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ - + payload_hash + "\nx-amz-date:" + date3 + "\n"; - string signed_headers = "host;x-amz-content-sha256;x-amz-date"; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("GET", "", "", date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - //DPRN("SHUNDEBUG1SIGN %s", string("Date:"+date3).c_str()); - // step 1 method : PARAM1: 'GET' - // step 2 c..uri : fixed: '/' - // step 3 query string: '' - // step 4 c..headers : PARAM3: 3 - // step 5 s..headers: PARAM5: 3 - // step 6 payload_hash : PARAM4: emptyhash - // step 7 c..request in calcsignature 1+...+6 - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("GET", "", date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("GET", "", "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2862,43 +2607,33 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query) responseHeaders.clear(); bodydata = new BodyData(); -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("GET", "", "", date, (resource + "/"))).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=query; - //DPRN("SHUNDEBUG2SIGN %s\t%s", string("CURL:"+ canonical_uri).c_str(), tpath); - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string signed_headers = "host;x-amz-content-sha256;x-amz-date"; - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ - + payload_hash + "\nx-amz-date:" + date3 + "\n"; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("GET", "", "", date, (resource + "/")); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - //DPRN("SHUNDEBUG2SIGN %s", string("Date:"+date3).c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("GET", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("GET", string(query), "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -2934,35 +2669,32 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); -#ifdef SIGV3 - turl += "?uploads"; - resource += "?uploads"; - url = prepare_url(turl.c_str()); - path = tpath; - requestHeaders = NULL; + string query_string = "uploads"; + if(!S3fsCurl::is_sigv4){ + query_string += "="; + } + turl += "?" + query_string; + resource += "?" + query_string; + url = prepare_url(turl.c_str()); + path = tpath; + requestHeaders = NULL; + bodydata = new BodyData(); responseHeaders.clear(); - bodydata = new BodyData(); - string date = get_date(); string contype = S3fsCurl::LookupMimeType(string(tpath)); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: "); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length: "); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Content-Type: " + contype).c_str()); for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - - if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ + string key = lower(iter->first); + string value = iter->second; + if(key.substr(0, 9) == "x-amz-acl"){ // not set value, but after set it. - }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ + }else if(key.substr(0, 10) == "x-amz-meta"){ + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); + }else if(key == "x-amz-server-side-encryption"){ // skip this header, because this header is specified after logic. - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ + }else if(key == "x-amz-server-side-encryption-customer-algorithm"){ // skip this header, because this header is specified with "x-amz-...-customer-key-md5". - }else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ + }else if(is_copy && key == "x-amz-server-side-encryption-customer-key-md5"){ // Only copy mode. if(!AddSseKeyRequestHead(value, is_copy)){ DPRNNN("Failed to insert sse(-c) header."); @@ -2970,12 +2702,12 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string } } // "x-amz-acl", rrs, sse - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.c_str()); if(S3fsCurl::is_use_rrs){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "REDUCED_REDUNDANCY"); } if(S3fsCurl::is_use_sse){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256"); }else if(S3fsCurl::IsSseCustomMode()){ string md5; if(!AddSseKeyRequestHead(md5, false)){ @@ -2986,110 +2718,41 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string // set additional header by ahbe conf requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); } - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("POST", "", contype, date, resource)).c_str()); - } -#else - string query_string = "uploads="; - turl += "?"+query_string; - resource += "?"+query_string; - url = prepare_url(turl.c_str()); - path = tpath; - requestHeaders = NULL; - responseHeaders.clear(); - bodydata = new BodyData(); - DPRN("SHUNDEBUG9POST %s", tpath); //string("Date:"+date3).c_str()); - // to be done: what's the payload_hash value? - string payload_hash = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=tpath; - string contype = S3fsCurl::LookupMimeType(string(tpath)); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; - string signed_headers = "host"; + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length", NULL); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); - // "x-amz-acl" - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); - canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n"; - signed_headers += ";x-amz-acl"; + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("POST", "", contype, date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; - signed_headers += ";x-amz-content-sha256"; + }else{ + // to be done: what's the payload_hash value? + string payload_hash = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length", NULL); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: "); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length: "); - - requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type: " + contype).c_str()); - canonical_headers.insert(0, "content-type:" + contype +"\n"); - signed_headers.insert(0, "content-type;"); - - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); - canonical_headers += "x-amz-date:" + date3 + "\n"; - signed_headers += ";x-amz-date"; - - for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - - if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){ - // not set value, but after set it. - }else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - canonical_headers += key + ":" + value + "\n"; - signed_headers += ";" + key; - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){ - // skip this header, because this header is specified after logic. - }else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){ - // skip this header, because this header is specified with "x-amz-...-customer-key-md5". - }else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){ - // Only copy mode. - if(!AddSseKeyRequestHead(value, is_copy)){ - DPRNNN("Failed to insert sse(-c) header."); - } + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("POST", tpath, query_string, strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); } } - // rrs, sse - if(S3fsCurl::is_use_sse){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256"); - canonical_headers += "x-amz-server-side-encryption:AES256\n"; - signed_headers += ";x-amz-server-side-encryption"; - }else if(S3fsCurl::IsSseCustomMode()){ - string md5; - if(!AddSseKeyRequestHead(md5, false)){ - DPRNNN("Failed to insert sse(-c) header."); - } - } - if(S3fsCurl::is_use_rrs){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); - canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n"; - signed_headers += ";x-amz-storage-class"; - } - if(is_use_ahbe){ - // set additional header by ahbe conf - requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); - } - - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignatureReal("POST", canonical_uri, query_string, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); - } -#endif - // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_POST, true); // POST @@ -3156,87 +2819,60 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id, string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); -#ifdef SIGV3 - turl += "?uploadId=" + upload_id; - resource += "?uploadId=" + upload_id; - url = prepare_url(turl.c_str()); - path = tpath; - requestHeaders = NULL; + string query_string = "uploadId=" + upload_id; + turl += "?" + query_string; + resource += "?" + query_string; + url = prepare_url(turl.c_str()); + path = tpath; + requestHeaders = NULL; + bodydata = new BodyData(); responseHeaders.clear(); - bodydata = new BodyData(); + string contype = S3fsCurl::LookupMimeType(string(tpath)); - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept:"); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type:"); + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("POST", "", "", date, resource)).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("POST", "", "", date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } + + }else{ + // to be done: what's the payload_hash value? + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + const unsigned char* cRequest = reinterpret_cast(postContent.c_str()); + unsigned int cRequest_len = postContent.size(); + unsigned char* sRequest = NULL; + unsigned int sRequest_len = 0; + char hexsRequest[64 + 1]; + unsigned int cnt; + s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len); + for(cnt = 0; cnt < sRequest_len; cnt++){ + sprintf(&hexsRequest[cnt * 2], "%02x", sRequest[cnt]); + } + payload_hash.assign(hexsRequest, &hexsRequest[sRequest_len * 2]); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", get_date_rfc850().c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("POST", tpath, query_string, strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#else - string query_string; - query_string = "uploadId=" + upload_id; - turl += "?" + query_string; - resource += "?" + query_string; - url = prepare_url(turl.c_str()); - path = tpath; - requestHeaders = NULL; - responseHeaders.clear(); - bodydata = new BodyData(); - - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - DPRN("SHUNDEBUG9POST %s", tpath); //string("Date:"+date3).c_str()); - - // to be done: what's the payload_hash value? - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - - unsigned char * cRequest = (unsigned char *)postContent.c_str(), *sRequest; - unsigned int cRequest_len= postContent.size(); - unsigned int sRequest_len = 0; - char hexsRequest[64]; - unsigned int i; - s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len); - for (i=0;i < sRequest_len;i++) sprintf(hexsRequest+(i*2), "%02x", sRequest[i]); - - - payload_hash.assign(hexsRequest, hexsRequest+sRequest_len*2); - - string canonical_uri = ""; - canonical_uri +=tpath; - string contype = S3fsCurl::LookupMimeType(string(tpath)); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; - string signed_headers = "host"; - - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept:"); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type: " + contype).c_str()); - canonical_headers.insert(0, "content-type:" + contype +"\n"); - signed_headers.insert(0, "content-type;"); - - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; - signed_headers += ";x-amz-content-sha256"; - - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); - canonical_headers += "x-amz-date:" + date3 + "\n"; - signed_headers += ";x-amz-date"; - - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignatureReal("POST", canonical_uri, query_string, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); - } -#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -3278,40 +2914,33 @@ int S3fsCurl::MultipartListRequest(string& body) responseHeaders.clear(); bodydata = new BodyData(); -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: "); + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("GET", "", "", date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ - + payload_hash + "\nx-amz-date:" + date3 + "\n"; - string signed_headers = "host;x-amz-content-sha256;x-amz-date"; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("GET", "", "", date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - //DPRN("SHUNDEBUG4SIGN %s", string("Date:"+date3).c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("GET", "", date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("GET", "", "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -3354,39 +2983,32 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, string& upload_id) requestHeaders = NULL; responseHeaders.clear(); -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("DELETE", "", "", date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=tpath; - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \ - + payload_hash + "\nx-amz-date:" + date3 + "\n"; - string signed_headers = "host;x-amz-content-sha256;x-amz-date"; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + - "/" + date2 + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("DELETE", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("DELETE", "", "", date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } + + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("DELETE", tpath, "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); @@ -3433,80 +3055,48 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string& } // make request -#ifdef SIGV3 - string urlargs = "?partNumber=" + IntToStr(part_num) + "&uploadId=" + upload_id; - string resource; - string turl; - MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); - - resource += urlargs; - turl += urlargs; - url = prepare_url(turl.c_str()); - path = tpath; - requestHeaders = NULL; - responseHeaders.clear(); - bodydata = new BodyData(); - headdata = new BodyData(); - - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: "); - - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("PUT", "", "", date, resource)).c_str()); - } -#else string request_uri = "partNumber=" + IntToStr(part_num) + "&uploadId=" + upload_id; - string urlargs = "?" + request_uri; + string urlargs = "?" + request_uri; string resource; string turl; MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); - resource += urlargs; - turl += urlargs; - url = prepare_url(turl.c_str()); - path = tpath; - requestHeaders = NULL; + resource += urlargs; + turl += urlargs; + url = prepare_url(turl.c_str()); + path = tpath; + requestHeaders = NULL; + bodydata = new BodyData(); + headdata = new BodyData(); responseHeaders.clear(); - bodydata = new BodyData(); - headdata = new BodyData(); - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=tpath; - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); - string ContentType; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; - string signed_headers = "host"; + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("PUT", "", "", date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); + } - //payload_hash="AWS4-HMAC-SHA256-PAYLOAD"; - payload_hash = s3fs_sha256sum(partdata.fd, partdata.startpos, partdata.size); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; - signed_headers += ";x-amz-content-sha256"; + }else{ + string payload_hash = s3fs_sha256sum(partdata.fd, partdata.startpos, partdata.size); + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); - canonical_headers += "x-amz-date:" + date3 + "\n"; - signed_headers += ";x-amz-date"; + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); - DPRN("SHUNDEBUG7PUT %s", string("Date:"+date3).c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignatureReal("PUT", canonical_uri, request_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("PUT", tpath, request_uri, strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -3580,92 +3170,47 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par headdata = new BodyData(); // Make request headers -#ifdef SIGV3 - string date = get_date(); - requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); - string ContentType; for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - if(0 == strcasecmp(key.c_str(), "Content-Type")){ + string key = lower(iter->first); + string value = iter->second; + if(key == "content-type"){ ContentType = value; - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source-range")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); + }else if(key == "x-amz-copy-source"){ + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); + }else if(key == "x-amz-copy-source-range"){ + requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); } // NOTICE: x-amz-acl, x-amz-server-side-encryption is not set! } - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS " + AWSAccessKeyId + ":" + - CalcSignature("PUT", "", ContentType, date, resource)).c_str()); - } -#else - string date = get_date(); - string date2 = get_date2(); - string date3 = get_date3(); - string canonical_uri = ""; - canonical_uri +=path; - string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; - string ContentType; - requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str()); - string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n"; - string signed_headers = "host"; -/* not need - // "x-amz-acl" - requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); - canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n"; - signed_headers += ";x-amz-acl"; -*/ - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-content-sha256:" + payload_hash).c_str()); - canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n"; - signed_headers += ";x-amz-content-sha256"; + if(!S3fsCurl::is_sigv4){ + string date = get_date_rfc850(); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); - requestHeaders = curl_slist_sort_insert(requestHeaders, - string("x-amz-date:" + date3).c_str()); - canonical_headers += "x-amz-date:" + date3 + "\n"; - signed_headers += ";x-amz-date"; - - for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string key = (*iter).first; - string value = (*iter).second; - if(0 == strcasecmp(key.c_str(), "Content-Type")){ - ContentType = value; - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - canonical_headers.insert(0, "content-type:" + value + "\n"); - signed_headers.insert(0, "content-type;"); - }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n")); - signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key)); - }else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source-range")){ - requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str()); - canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n")); - signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key)); + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignatureV2("PUT", "", ContentType, date, resource); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); } - // NOTICE: x-amz-acl, x-amz-server-side-encryption is not set! - } - if(S3fsCurl::is_use_rrs){ - requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY"); - canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n"; - signed_headers += ";x-amz-storage-class"; - } - DPRN("SHUNDEBUG8PUT %s", string("Date:"+date3).c_str()); - if(!S3fsCurl::IsPublicBucket()){ - requestHeaders = curl_slist_sort_insert( - requestHeaders, - string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2 - + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" + - CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str()); + }else{ + string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + string strdate; + string date8601; + get_date_sigv3(strdate, date8601); + + requestHeaders = curl_slist_sort_insert(requestHeaders, "host", string(bucket + ".s3.amazonaws.com").c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-content-sha256", payload_hash.c_str()); + requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); + + if(!S3fsCurl::IsPublicBucket()){ + string Signature = CalcSignature("PUT", path, "", strdate, payload_hash, date8601); + requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", + string("AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + + "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature).c_str()); + } } -#endif // setopt curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); @@ -4280,9 +3825,8 @@ struct curl_slist* AdditionalHeader::AddHeader(struct curl_slist* list, const ch return list; } for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ - string slistval = (*iter).first + ":" + (*iter).second; // Adding header - list = curl_slist_sort_insert(list, slistval.c_str()); + list = curl_slist_sort_insert(list, iter->first.c_str(), iter->second.c_str()); } meta.clear(); S3FS_MALLOCTRIM(0); @@ -4330,36 +3874,53 @@ bool AdditionalHeader::Dump(void) const // Because AWS signature needs sorted header. // struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* data) +{ + if(!data){ + return list; + } + string strkey = data; + string strval = ""; + + string::size_type pos = strkey.find(':', 0); + if(string::npos != pos){ + strval = strkey.substr(pos + 1); + strkey = strkey.substr(0, pos); + } + + return curl_slist_sort_insert(list, strkey.c_str(), strval.c_str()); +} + +struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value) { struct curl_slist* curpos; struct curl_slist* lastpos; struct curl_slist* new_item; - if(!data){ + if(!key){ return list; } if(NULL == (new_item = (struct curl_slist*)malloc(sizeof(struct curl_slist)))){ return list; } - if(NULL == (new_item->data = strdup(data))){ + + // key & value are trimed and lower(only key) + string strkey = trim(string(key)); + string strval = trim(string(value ? value : "")); + string strnew = key + string(": ") + strval; + if(NULL == (new_item->data = strdup(strnew.c_str()))){ free(new_item); return list; } new_item->next = NULL; - string strnew = data; - string::size_type pos = strnew.find(':', 0); - if(string::npos != pos){ - strnew = strnew.substr(0, pos); - } - - for(lastpos = NULL, curpos = list; curpos; curpos = curpos->next){ + for(lastpos = NULL, curpos = list; curpos; lastpos = curpos, curpos = curpos->next){ string strcur = curpos->data; + size_t pos; if(string::npos != (pos = strcur.find(':', 0))){ strcur = strcur.substr(0, pos); } - int result = strcmp(strnew.c_str(), strcur.c_str()); + int result = strcasecmp(strkey.c_str(), strcur.c_str()); if(0 == result){ // same data, so replace it. if(lastpos){ @@ -4382,8 +3943,8 @@ struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* d new_item->next = curpos; break; } - lastpos = curpos; } + if(!curpos){ // append to last pos if(lastpos){ @@ -4393,9 +3954,86 @@ struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* d list = new_item; } } + return list; } +string get_sorted_header_keys(const struct curl_slist* list) +{ + string sorted_headers; + + if(!list){ + return sorted_headers; + } + + for( ; list; list = list->next){ + string strkey = list->data; + size_t pos; + if(string::npos != (pos = strkey.find(':', 0))){ + strkey = strkey.substr(0, pos); + } + if(0 < sorted_headers.length()){ + sorted_headers += ";"; + } + sorted_headers += lower(strkey); + } + + return sorted_headers; +} + +string get_canonical_headers(const struct curl_slist* list) +{ + string canonical_headers; + + if(!list){ + canonical_headers = "\n"; + return canonical_headers; + } + + for( ; list; list = list->next){ + string strhead = list->data; + size_t pos; + if(string::npos != (pos = strhead.find(':', 0))){ + string strkey = trim(lower(strhead.substr(0, pos))); + string strval = trim(strhead.substr(pos + 1)); + strhead = strkey + string(":") + strval; + }else{ + strhead = trim(lower(strhead)); + } + canonical_headers += strhead; + canonical_headers += "\n"; + } + return canonical_headers; +} + +string get_canonical_headers(const struct curl_slist* list, bool only_amz) +{ + string canonical_headers; + + if(!list){ + canonical_headers = "\n"; + return canonical_headers; + } + + for( ; list; list = list->next){ + string strhead = list->data; + size_t pos; + if(string::npos != (pos = strhead.find(':', 0))){ + string strkey = trim(lower(strhead.substr(0, pos))); + string strval = trim(strhead.substr(pos + 1)); + strhead = strkey + string(":") + strval; + }else{ + strhead = trim(lower(strhead)); + } + if(only_amz && strhead.substr(0, 5) != "x-amz"){ + continue; + } + canonical_headers += strhead; + canonical_headers += "\n"; + } + return canonical_headers; +} + // function for using global values bool MakeUrlResource(const char* realpath, string& resourcepath, string& url) { diff --git a/src/curl.h b/src/curl.h index 1430edf..0bc0e97 100644 --- a/src/curl.h +++ b/src/curl.h @@ -181,6 +181,7 @@ class S3fsCurl static mimes_t mimeTypes; static int max_parallel_cnt; static off_t multipart_size; + static bool is_sigv4; // variables CURL* hCurl; @@ -244,15 +245,8 @@ class S3fsCurl bool ResetHandle(void); bool RemakeHandle(void); bool ClearInternalData(void); -#ifdef SIGV3 - std::string CalcSignature(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource); -#else - std::string CalcSignaturev2(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource); - std::string CalcSignature(std::string method, std::string canonical_uri, std::string date2, std::string cononical_headers, - std::string payload_hash, std::string signed_headers, std::string date3); - std::string CalcSignatureReal(std::string method, std::string canonical_uri, std::string query_string, std::string date2, std::string cononical_headers, - std::string payload_hash, std::string signed_headers, std::string date3); -#endif + std::string CalcSignatureV2(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource); + std::string CalcSignature(std::string method, std::string canonical_uri, std::string query_string, std::string strdate, std::string payload_hash, std::string date8601); bool GetUploadId(std::string& upload_id); int GetIAMCredentials(void); @@ -306,6 +300,8 @@ class S3fsCurl static const char* GetIAMRole(void) { return S3fsCurl::IAM_role.c_str(); } static bool SetMultipartSize(off_t size); static off_t GetMultipartSize(void) { return S3fsCurl::multipart_size; } + static bool SetSignatureV4(bool isset = true) { bool bresult = S3fsCurl::is_sigv4; S3fsCurl::is_sigv4 = isset; return bresult; } + static bool IsSignatureV4(void) { return S3fsCurl::is_sigv4; } // methods bool CreateCurlHandle(bool force = false); @@ -429,6 +425,9 @@ std::string GetContentMD5(int fd); unsigned char* md5hexsum(int fd, off_t start, ssize_t size); std::string md5sum(int fd, off_t start, ssize_t size); struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* data); +struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value); +std::string get_sorted_header_keys(const struct curl_slist* list); +std::string get_canonical_headers(const struct curl_slist* list, bool only_amz = false); bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url); std::string prepare_url(const char* url); diff --git a/src/openssl_auth.cpp b/src/openssl_auth.cpp index cafd41b..2fbe8a0 100644 --- a/src/openssl_auth.cpp +++ b/src/openssl_auth.cpp @@ -32,9 +32,7 @@ #include #include #include -#ifndef SIGV3 #include -#endif #include #include #include @@ -185,7 +183,7 @@ bool s3fs_destroy_crypt_mutex(void) //------------------------------------------------------------------- // Utility Function for HMAC //------------------------------------------------------------------- -bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen, bool is_sha256) { if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){ return false; @@ -194,15 +192,25 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){ return false; } -#ifdef SIGV3 - HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen); -#else - HMAC(EVP_sha256(), key, keylen, data, datalen, *digest, digestlen); -#endif + if(is_sha256){ + HMAC(EVP_sha256(), key, keylen, data, datalen, *digest, digestlen); + }else{ + HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen); + } return true; } +bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, false); +} + +bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, true); +} + //------------------------------------------------------------------- // Utility Function for MD5 //------------------------------------------------------------------- @@ -262,7 +270,6 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size) return result; } -#ifndef SIGV3 //------------------------------------------------------------------- // Utility Function for SHA256 //------------------------------------------------------------------- @@ -274,12 +281,12 @@ size_t get_sha256_digest_length(void) bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen) { (*digestlen) = EVP_MAX_MD_SIZE * sizeof(unsigned char); - if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){ + if(NULL == ((*digest) = reinterpret_cast(malloc(*digestlen)))){ return false; } - const EVP_MD *md = EVP_get_digestbyname("sha256"); - EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + const EVP_MD* md = EVP_get_digestbyname("sha256"); + EVP_MD_CTX* mdctx = EVP_MD_CTX_create(); EVP_DigestInit_ex(mdctx, md, NULL); EVP_DigestUpdate(mdctx, data, datalen); EVP_DigestFinal_ex(mdctx, *digest, digestlen); @@ -290,16 +297,15 @@ bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char* unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) { - - const EVP_MD *md = EVP_get_digestbyname("sha256"); - EVP_MD_CTX *sha256ctx = EVP_MD_CTX_create(); + const EVP_MD* md = EVP_get_digestbyname("sha256"); + EVP_MD_CTX* sha256ctx = EVP_MD_CTX_create(); EVP_DigestInit_ex(sha256ctx, md, NULL); - char buf[512]; - ssize_t bytes; + char buf[512]; + ssize_t bytes; unsigned char* result; - if(-1 == size) { + if(-1 == size){ struct stat st; if(-1 == fstat(fd, &st)){ return NULL; @@ -339,7 +345,6 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) } return result; } -#endif /* * Local variables: diff --git a/src/s3fs.cpp b/src/s3fs.cpp index e69f5fd..84589e6 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -88,9 +88,7 @@ std::string program_name; std::string service_path = "/"; std::string host = "http://s3.amazonaws.com"; std::string bucket = ""; -#ifndef SIGV3 std::string endpoint = "us-east-1"; -#endif //------------------------------------------------------------------- // Static valiables @@ -3767,12 +3765,14 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar } return 0; } -#ifndef SIGV3 + if(0 == strcmp(arg, "sigv2")){ + S3fsCurl::SetSignatureV4(false); + return 0; + } if(0 == STR2NCMP(arg, "endpoint=")){ endpoint = strchr(arg, '=') + sizeof(char); return 0; } -#endif if(0 == strcmp(arg, "use_path_request_style")){ pathrequeststyle = true; return 0; diff --git a/src/s3fs_auth.h b/src/s3fs_auth.h index fb3fad1..6893db5 100644 --- a/src/s3fs_auth.h +++ b/src/s3fs_auth.h @@ -29,9 +29,7 @@ char* s3fs_base64(unsigned char* input, size_t length); std::string s3fs_get_content_md5(int fd); std::string s3fs_md5sum(int fd, off_t start, ssize_t size); -#ifndef SIGV3 std::string s3fs_sha256sum(int fd, off_t start, ssize_t size); -#endif // // in xxxxxx_auth.cpp @@ -42,13 +40,12 @@ bool s3fs_destroy_global_ssl(void); bool s3fs_init_crypt_mutex(void); bool s3fs_destroy_crypt_mutex(void); bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen); +bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen); size_t get_md5_digest_length(void); unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size); -#ifndef SIGV3 bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen); size_t get_sha256_digest_length(void); unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size); -#endif #endif // S3FS_AUTH_H_ diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index 43b889e..f481389 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -975,11 +975,12 @@ void show_help (void) "\n" " url (default=\"http://s3.amazonaws.com\")\n" " - sets the url to use to access amazon s3\n" -#ifndef SIGV3 "\n" " endpoint (default=\"us-east-1\")\n" " - sets the endpoint to use\n" -#endif + "\n" + " sigv2 (default is signature version 4)\n" + " - sets signing AWS requests by sing Signature Version 2\n" "\n" " nomultipart (disable multipart uploads)\n" "\n" diff --git a/src/string_util.cpp b/src/string_util.cpp index c3efe1f..15385c6 100644 --- a/src/string_util.cpp +++ b/src/string_util.cpp @@ -125,12 +125,10 @@ string urlEncode(const string &s) for (unsigned i = 0; i < s.length(); ++i) { if (s[i] == '/') { // Note- special case for fuse paths... result += s[i]; -#ifndef SIGV3 }else if (s[i] == '=') { // Note- special case for s3... result += s[i]; }else if (s[i] == '&') { // Note- special case for s3... result += s[i]; -#endif } else if (isalnum(s[i])) { result += s[i]; } else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') { @@ -145,11 +143,9 @@ string urlEncode(const string &s) result += hexAlphabet[static_cast(s[i]) % 16]; } } - return result; } -#ifndef SIGV3 /** * urlEncode a fuse path, * taking into special consideration "/", @@ -177,10 +173,8 @@ string urlEncode2(const string &s) result += hexAlphabet[static_cast(s[i]) % 16]; } } - return result; } -#endif // // ex. target="http://......?keyword=value&..." @@ -212,7 +206,7 @@ bool get_keyword_value(string& target, const char* keyword, string& value) * Returns the current date * in a format suitable for a HTTP request header. */ -string get_date() +string get_date_rfc850() { char buf[100]; time_t t = time(NULL); @@ -220,25 +214,27 @@ string get_date() return buf; } -#ifndef SIGV3 -string get_date2() +void get_date_sigv3(string& date, string& date8601) +{ + time_t tm = time(NULL); + date = get_date_string(tm); + date8601 = get_date_iso8601(tm); +} + +string get_date_string(time_t tm) { char buf[100]; - time_t t = time(NULL); - strftime(buf, sizeof(buf), "%Y%m%d", gmtime(&t)); + strftime(buf, sizeof(buf), "%Y%m%d", gmtime(&tm)); return buf; } -string get_date3() +string get_date_iso8601(time_t tm) { char buf[100]; - time_t t = time(NULL); - strftime(buf, sizeof(buf), "%Y%m%dT%H%M%SZ", gmtime(&t)); + strftime(buf, sizeof(buf), "%Y%m%dT%H%M%SZ", gmtime(&tm)); return buf; } -#endif - /* * Local variables: * tab-width: 4 diff --git a/src/string_util.h b/src/string_util.h index cad1239..4774674 100644 --- a/src/string_util.h +++ b/src/string_util.h @@ -46,15 +46,12 @@ std::string trim_right(const std::string &s, const std::string &t = SPACES); std::string trim(const std::string &s, const std::string &t = SPACES); std::string lower(std::string s); std::string IntToStr(int); -std::string get_date(); -#ifndef SIGV3 -std::string get_date2(); -std::string get_date3(); -#endif +std::string get_date_rfc850(void); +void get_date_sigv3(std::string& date, std::string& date8601); +std::string get_date_string(time_t tm); +std::string get_date_iso8601(time_t tm); std::string urlEncode(const std::string &s); -#ifndef SIGV3 std::string urlEncode2(const std::string &s); -#endif bool get_keyword_value(std::string& target, const char* keyword, std::string& value); #endif // S3FS_STRING_UTIL_H_ From 1424f877544aee7b279e1aac4bc5ecaeb0bae8c2 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Mon, 2 Feb 2015 16:36:08 +0000 Subject: [PATCH 4/4] Supported signature version 4 for GnuTLS/NSS and automatically set endpoint/sigv2 --- doc/man/s3fs.1 | 4 + src/curl.cpp | 3 - src/gnutls_auth.cpp | 187 ++++++++++++++++++++++++++++++++++++++++++++ src/nss_auth.cpp | 98 ++++++++++++++++++++++- src/s3fs.cpp | 90 ++++++++++++++++++++- src/s3fs_util.cpp | 9 ++- 6 files changed, 381 insertions(+), 10 deletions(-) diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index 438bf0a..c0f11aa 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -151,6 +151,10 @@ sets the url to use to access Amazon S3. If you want to use HTTPS, then you can .TP \fB\-o\fR endpoint (default="us-east-1") sets the endpoint to use. +If this option is not specified, s3fs uses \"us-east-1\" region as the default. +If the s3fs could not connect to the region specified by this option, s3fs could not run. +But if you do not specify this option, and if you can not connect with the default region, s3fs will retry to automatically connect to the other region. +So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server. .TP \fB\-o\fR sigv2 (default is signature version 4) sets signing AWS requests by sing Signature Version 2. diff --git a/src/curl.cpp b/src/curl.cpp index 2fcec94..7fa85ee 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -2577,9 +2577,6 @@ int S3fsCurl::CheckBucket(void) if (result != 0) { DPRN("Check bucket failed, S3 response: %s", (bodydata ? bodydata->str() : "")); } - delete bodydata; - bodydata = NULL; - return result; } diff --git a/src/gnutls_auth.cpp b/src/gnutls_auth.cpp index 72de2df..9ffe958 100644 --- a/src/gnutls_auth.cpp +++ b/src/gnutls_auth.cpp @@ -120,6 +120,25 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t return true; } +bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){ + return false; + } + + if(NULL == (*digest = (unsigned char*)malloc(SHA256_DIGEST_SIZE))){ + return false; + } + + struct hmac_sha256_ctx ctx_hmac; + hmac_sha256_set_key(&ctx_hmac, keylen, reinterpret_cast(key)); + hmac_sha256_update(&ctx_hmac, datalen, reinterpret_cast(data)); + hmac_sha256_digest(&ctx_hmac, SHA256_DIGEST_SIZE, reinterpret_cast(*digest)); + *digestlen = SHA256_DIGEST_SIZE; + + return true; +} + #else // USE_GNUTLS_NETTLE bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) @@ -142,6 +161,26 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t return true; } +bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){ + return false; + } + + if(0 >= (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA256))){ + return false; + } + if(NULL == (*digest = (unsigned char*)malloc(*digestlen + 1))){ + return false; + } + if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA256, key, keylen, data, datalen, *digest)){ + free(*digest); + *digest = NULL; + return false; + } + return true; +} + #endif // USE_GNUTLS_NETTLE //------------------------------------------------------------------- @@ -256,6 +295,154 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size) #endif // USE_GNUTLS_NETTLE +//------------------------------------------------------------------- +// Utility Function for SHA256 +//------------------------------------------------------------------- +#define SHA256_DIGEST_LENGTH 32 + +size_t get_sha256_digest_length(void) +{ + return SHA256_DIGEST_LENGTH; +} + +#ifdef USE_GNUTLS_NETTLE +bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen) +{ + (*digestlen) = static_cast(get_sha256_digest_length()); + if(NULL == ((*digest) = reinterpret_cast(malloc(*digestlen)))){ + return false; + } + + struct sha256_ctx ctx_sha256; + sha256_init(&ctx_sha256); + sha256_update(&ctx_sha256, datalen, data); + sha256_digest(&ctx_sha256, *digestlen, *digest); + + return true; +} + +unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) +{ + struct sha256_ctx ctx_sha256; + unsigned char buf[512]; + ssize_t bytes; + unsigned char* result; + + // seek to top of file. + if(-1 == lseek(fd, start, SEEK_SET)){ + return NULL; + } + + memset(buf, 0, 512); + sha256_init(&ctx_sha256); + + for(ssize_t total = 0; total < size; total += bytes){ + bytes = 512 < (size - total) ? 512 : (size - total); + bytes = read(fd, buf, bytes); + if(0 == bytes){ + // end of file + break; + }else if(-1 == bytes){ + // error + DPRNNN("file read error(%d)", errno); + return NULL; + } + sha256_update(&ctx_sha256, bytes, buf); + memset(buf, 0, 512); + } + if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){ + return NULL; + } + sha256_digest(&ctx_sha256, get_sha256_digest_length(), result); + + if(-1 == lseek(fd, start, SEEK_SET)){ + free(result); + return NULL; + } + + return result; +} + +#else // USE_GNUTLS_NETTLE + +bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen) +{ + (*digestlen) = static_cast(get_sha256_digest_length()); + if(NULL == ((*digest) = reinterpret_cast(malloc(*digestlen)))){ + return false; + } + + gcry_md_hd_t ctx_sha256; + gcry_error_t err; + if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){ + DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); + free(*digest); + return false; + } + gcry_md_write(ctx_sha256, data, datalen); + memcpy(*digest, gcry_md_read(ctx_sha256, 0), *digestlen); + gcry_md_close(ctx_sha256); + + return true; +} + +unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) +{ + gcry_md_hd_t ctx_sha256; + gcry_error_t err; + char buf[512]; + ssize_t bytes; + unsigned char* result; + + if(-1 == size){ + struct stat st; + if(-1 == fstat(fd, &st)){ + return NULL; + } + size = static_cast(st.st_size); + } + + // seek to top of file. + if(-1 == lseek(fd, start, SEEK_SET)){ + return NULL; + } + + memset(buf, 0, 512); + if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){ + DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); + return NULL; + } + + for(ssize_t total = 0; total < size; total += bytes){ + bytes = 512 < (size - total) ? 512 : (size - total); + bytes = read(fd, buf, bytes); + if(0 == bytes){ + // end of file + break; + }else if(-1 == bytes){ + // error + DPRNNN("file read error(%d)", errno); + return NULL; + } + gcry_md_write(ctx_sha256, buf, bytes); + memset(buf, 0, 512); + } + if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){ + return NULL; + } + memcpy(result, gcry_md_read(ctx_sha256, 0), get_sha256_digest_length()); + gcry_md_close(ctx_sha256); + + if(-1 == lseek(fd, start, SEEK_SET)){ + free(result); + return NULL; + } + + return result; +} + +#endif // USE_GNUTLS_NETTLE + /* * Local variables: * tab-width: 4 diff --git a/src/nss_auth.cpp b/src/nss_auth.cpp index f820426..4fb11a5 100644 --- a/src/nss_auth.cpp +++ b/src/nss_auth.cpp @@ -83,7 +83,7 @@ bool s3fs_destroy_crypt_mutex(void) //------------------------------------------------------------------- // Utility Function for HMAC //------------------------------------------------------------------- -bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen, bool is_sha256) { if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){ return false; @@ -94,17 +94,17 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t PK11Context* Context; SECStatus SecStatus; unsigned char tmpdigest[64]; - SECItem KeySecItem = {siBuffer, reinterpret_cast(const_cast(key)), keylen}; + SECItem KeySecItem = {siBuffer, reinterpret_cast(const_cast(key)), static_cast(keylen)}; SECItem NullSecItem = {siBuffer, NULL, 0}; if(NULL == (Slot = PK11_GetInternalKeySlot())){ return false; } - if(NULL == (pKey = PK11_ImportSymKey(Slot, CKM_SHA_1_HMAC, PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){ + if(NULL == (pKey = PK11_ImportSymKey(Slot, (is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){ PK11_FreeSlot(Slot); return false; } - if(NULL == (Context = PK11_CreateContextBySymKey(CKM_SHA_1_HMAC, CKA_SIGN, pKey, &NullSecItem))){ + if(NULL == (Context = PK11_CreateContextBySymKey((is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), CKA_SIGN, pKey, &NullSecItem))){ PK11_FreeSymKey(pKey); PK11_FreeSlot(Slot); return false; @@ -132,6 +132,16 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t return true; } +bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, false); +} + +bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, true); +} + //------------------------------------------------------------------- // Utility Function for MD5 //------------------------------------------------------------------- @@ -193,6 +203,86 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size) return result; } +//------------------------------------------------------------------- +// Utility Function for SHA256 +//------------------------------------------------------------------- +size_t get_sha256_digest_length(void) +{ + return SHA256_LENGTH; +} + +bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen) +{ + (*digestlen) = static_cast(get_sha256_digest_length()); + if(NULL == ((*digest) = reinterpret_cast(malloc(*digestlen)))){ + return false; + } + + PK11Context* sha256ctx; + unsigned int sha256outlen; + sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256); + + PK11_DigestOp(sha256ctx, data, datalen); + PK11_DigestFinal(sha256ctx, *digest, &sha256outlen, *digestlen); + PK11_DestroyContext(sha256ctx, PR_TRUE); + *digestlen = sha256outlen; + + return true; +} + +unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) +{ + PK11Context* sha256ctx; + unsigned char buf[512]; + ssize_t bytes; + unsigned char* result; + unsigned int sha256outlen; + + if(-1 == size){ + struct stat st; + if(-1 == fstat(fd, &st)){ + return NULL; + } + size = static_cast(st.st_size); + } + + // seek to top of file. + if(-1 == lseek(fd, start, SEEK_SET)){ + return NULL; + } + + memset(buf, 0, 512); + sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256); + + for(ssize_t total = 0; total < size; total += bytes){ + bytes = 512 < (size - total) ? 512 : (size - total); + bytes = read(fd, buf, bytes); + if(0 == bytes){ + // end of file + break; + }else if(-1 == bytes){ + // error + DPRNNN("file read error(%d)", errno); + return NULL; + } + PK11_DigestOp(sha256ctx, buf, bytes); + memset(buf, 0, 512); + } + if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){ + PK11_DestroyContext(sha256ctx, PR_TRUE); + return NULL; + } + PK11_DigestFinal(sha256ctx, result, &sha256outlen, get_sha256_digest_length()); + PK11_DestroyContext(sha256ctx, PR_TRUE); + + if(-1 == lseek(fd, start, SEEK_SET)){ + free(result); + return NULL; + } + + return result; +} + /* * Local variables: * tab-width: 4 diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 84589e6..5bef1bb 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -84,6 +84,7 @@ bool foreground = false; bool foreground2 = false; bool nomultipart = false; bool pathrequeststyle = false; +bool is_specified_endpoint = false; std::string program_name; std::string service_path = "/"; std::string host = "http://s3.amazonaws.com"; @@ -2979,6 +2980,41 @@ static int s3fs_utility_mode(void) return result; } +// +// If calling with wrong region, s3fs gets following error body as 400 erro code. +// "AuthorizationHeaderMalformedThe authorization header is +// malformed; the region 'us-east-1' is wrong; expecting 'ap-northeast-1' +// ap-northeast-1...... +// " +// +// So this is cheep codes but s3fs should get correct reagion automatically. +// +static bool check_region_error(const char* pbody, string& expectregion) +{ + if(!pbody){ + return false; + } + const char* region; + const char* regionend; + if(NULL == (region = strcasestr(pbody, "The authorization header is malformed; the region "))){ + return false; + } + if(NULL == (region = strcasestr(region, "expecting \'"))){ + return false; + } + region += strlen("expecting \'"); + if(NULL == (regionend = strchr(region, '\''))){ + return false; + } + string strtmp(region, (regionend - region)); + if(0 == strtmp.length()){ + return false; + } + expectregion = strtmp; + + return true; +} + static int s3fs_check_service(void) { FPRN("check services."); @@ -2990,12 +3026,61 @@ static int s3fs_check_service(void) } S3fsCurl s3fscurl; - if(0 != s3fscurl.CheckBucket()){ + if(-1 == s3fscurl.CheckBucket()){ fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str()); return EXIT_FAILURE; } long responseCode = s3fscurl.GetLastResponseCode(); + if(responseCode == 400){ + if(!S3fsCurl::IsSignatureV4()){ + // signature version 2 + fprintf(stderr, "%s: Bad Request\n", program_name.c_str()); + return EXIT_FAILURE; + } + if(is_specified_endpoint){ + // if specifies endpoint, do not retry to connect. + fprintf(stderr, "%s: Bad Request\n", program_name.c_str()); + return EXIT_FAILURE; + } + + // check region error for signature version 4 + BodyData* body = s3fscurl.GetBodyData(); + string expectregion; + if(check_region_error(body->str(), expectregion)){ + // not specified endpoint, so try to connect to expected region. + LOWSYSLOGPRINT(LOG_ERR, "Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str()); + FPRN("Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str()); + endpoint = expectregion; + + // retry to check + s3fscurl.DestroyCurlHandle(); + if(-1 == s3fscurl.CheckBucket()){ + fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str()); + return EXIT_FAILURE; + } + responseCode = s3fscurl.GetLastResponseCode(); + } + + if(responseCode == 400){ + // retry to use sigv2 + LOWSYSLOGPRINT(LOG_ERR, "Could not connect, so retry to connect by signature version 2."); + FPRN("Could not connect, so retry to connect by signature version 2."); + S3fsCurl::SetSignatureV4(); + + // retry to check + s3fscurl.DestroyCurlHandle(); + if(-1 == s3fscurl.CheckBucket()){ + fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str()); + return EXIT_FAILURE; + } + responseCode = s3fscurl.GetLastResponseCode(); + if(responseCode == 400){ + fprintf(stderr, "%s: Bad Request\n", program_name.c_str()); + return EXIT_FAILURE; + } + } + } if(responseCode == 403){ fprintf(stderr, "%s: invalid credentials\n", program_name.c_str()); return EXIT_FAILURE; @@ -3770,7 +3855,8 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar return 0; } if(0 == STR2NCMP(arg, "endpoint=")){ - endpoint = strchr(arg, '=') + sizeof(char); + endpoint = strchr(arg, '=') + sizeof(char); + is_specified_endpoint = true; return 0; } if(0 == strcmp(arg, "use_path_request_style")){ diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index f481389..484d11a 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -977,7 +977,14 @@ void show_help (void) " - sets the url to use to access amazon s3\n" "\n" " endpoint (default=\"us-east-1\")\n" - " - sets the endpoint to use\n" + " - sets the endpoint to use on signatue version 4\n" + " If this option is not specified, s3fs uses \"us-east-1\" region as\n" + " the default. If the s3fs could not connect to the region specified\n" + " by this option, s3fs could not run. But if you do not specify this\n" + " option, and if you can not connect with the default region, s3fs\n" + " will retry to automatically connect to the other region. So s3fs\n" + " can know the correct region name, because s3fs can find it in an\n" + " error from the S3 server.\n" "\n" " sigv2 (default is signature version 4)\n" " - sets signing AWS requests by sing Signature Version 2\n"