Merged manually from caxapniy/s3fs-fuse/tree/1.77v4merge for signature v4 - #102

This commit is contained in:
Takeshi Nakatani 2015-01-20 16:31:36 +00:00
parent 98daf16681
commit bb1f1d3faa
10 changed files with 981 additions and 0 deletions

View File

@ -94,6 +94,9 @@ extern std::string service_path;
extern std::string host; extern std::string host;
extern std::string bucket; extern std::string bucket;
extern std::string mount_prefix; extern std::string mount_prefix;
#ifndef SIGV3
extern std::string endpoint;
#endif
#endif // S3FS_COMMON_H_ #endif // S3FS_COMMON_H_

View File

@ -102,6 +102,29 @@ string s3fs_md5sum(int fd, off_t start, ssize_t size)
return string(md5); return string(md5);
} }
#ifndef SIGV3
string s3fs_sha256sum(int fd, off_t start, ssize_t size)
{
size_t digestlen = get_sha256_digest_length();
char sha256[2 * digestlen + 1];
char hexbuf[3];
unsigned char* sha256hex;
if(NULL == (sha256hex = s3fs_sha256hexsum(fd, start, size))){
return string("");
}
memset(sha256, 0, 2 * digestlen + 1);
for(size_t pos = 0; pos < digestlen; pos++){
snprintf(hexbuf, 3, "%02x", sha256hex[pos]);
strncat(sha256, hexbuf, 2);
}
free(sha256hex);
return string(sha256);
}
#endif
/* /*
* Local variables: * Local variables:
* tab-width: 4 * tab-width: 4

View File

@ -1742,6 +1742,7 @@ int S3fsCurl::RequestPerform(void)
// @param date e.g., get_date() // @param date e.g., get_date()
// @param resource e.g., "/pub" // @param resource e.g., "/pub"
// //
#ifdef SIGV3
string S3fsCurl::CalcSignature(string method, string strMD5, string content_type, string date, string resource) string S3fsCurl::CalcSignature(string method, string strMD5, string content_type, string date, string resource)
{ {
string Signature; string Signature;
@ -1789,6 +1790,131 @@ string S3fsCurl::CalcSignature(string method, string strMD5, string content_type
return Signature; return Signature;
} }
#else
// NOT SIGV3
string S3fsCurl::CalcSignaturev2(string method, string strMD5, string content_type, string date, string resource)
{
string Signature;
string StringToSign;
if(0 < S3fsCurl::IAM_role.size()){
if(!S3fsCurl::CheckIAMCredentialUpdate()){
DPRN("Something error occurred in checking IAM credential.");
return Signature; // returns empty string, then it occures error.
}
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str());
}
StringToSign += method + "\n";
StringToSign += strMD5 + "\n"; // md5
StringToSign += content_type + "\n";
StringToSign += date + "\n";
for(curl_slist* headers = requestHeaders; headers; headers = headers->next){
if(0 == strncmp(headers->data, "x-amz", 5)){
StringToSign += headers->data;
StringToSign += "\n";
}
}
StringToSign += resource;
const void* key = S3fsCurl::AWSSecretAccessKey.data();
int key_len = S3fsCurl::AWSSecretAccessKey.size();
const unsigned char* sdata = reinterpret_cast<const unsigned char*>(StringToSign.data());
int sdata_len = StringToSign.size();
unsigned char* md = NULL;
unsigned int md_len = 0;;
s3fs_HMAC(key, key_len, sdata, sdata_len, &md, &md_len);
char* base64;
if(NULL == (base64 = s3fs_base64(md, md_len))){
free(md);
return string(""); // ENOMEM
}
free(md);
Signature = base64;
free(base64);
return Signature;
}
string S3fsCurl::CalcSignatureReal(string method, string canonical_uri, string query_string , string date2,
string canonical_headers, string payload_hash, string signed_headers, string date3)
{
string Signature, StringCQ, StringToSign;
string uriencode;
if(0 < S3fsCurl::IAM_role.size()){
if(!S3fsCurl::CheckIAMCredentialUpdate()){
DPRN("Something error occurred in checking IAM credential.");
return Signature; // returns empty string, then it occures error.
}
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-security-token:" + S3fsCurl::AWSAccessToken).c_str());
}
uriencode = urlEncode(canonical_uri);
StringCQ = method + "\n";
if(0 == strcmp(method.c_str(),"HEAD") || 0 == strcmp(method.c_str(),"PUT") || 0 == strcmp(method.c_str(),"DELETE")){
StringCQ += uriencode + "\n" + query_string + "\n";
}else if (0 == strcmp(method.c_str(), "GET") && 0 == strcmp(uriencode.c_str(), "")) {
StringCQ +="/\n\n";
}else if (0 == strcmp(method.c_str(), "GET") && 0 == strncmp(uriencode.c_str(), "/",1)) {
StringCQ += uriencode +"\n\n";
}else if (0 == strcmp(method.c_str(), "GET") && 0 != strncmp(uriencode.c_str(), "/",1)) {
StringCQ += "/\n" + urlEncode2(canonical_uri) +"\n";
}else if (0 == strcmp(method.c_str(), "POST")) {
StringCQ += uriencode +"\n" + query_string +"\n";
}
StringCQ += canonical_headers + "\n";
StringCQ += signed_headers + "\n";
StringCQ += payload_hash;
unsigned char * cRequest = (unsigned char *)StringCQ.c_str();
unsigned int cRequest_len= StringCQ.size();
// DPRN("SHUNDEBUGXXXPUT: %s", cRequest);
char kSecret[128];
unsigned char *kDate, *kRegion, *kService, *kSigning, *sRequest = NULL;
unsigned int kDate_len,kRegion_len, kService_len, kSigning_len, sRequest_len = 0;
char hexsRequest[64];
int kSecret_len = snprintf(kSecret, sizeof(kSecret), "AWS4%s", S3fsCurl::AWSSecretAccessKey.c_str());
unsigned int i;
s3fs_HMAC(kSecret, kSecret_len, (unsigned char*)date2.data(), date2.size(), &kDate, &kDate_len);
s3fs_HMAC(kDate, kDate_len, (unsigned char *)endpoint.c_str(), endpoint.size(), &kRegion, &kRegion_len);
s3fs_HMAC(kRegion, kRegion_len, (unsigned char *)"s3", sizeof("s3")-1, &kService, &kService_len);
s3fs_HMAC(kService, kService_len, (unsigned char *)"aws4_request", sizeof("aws4_request")-1, &kSigning, &kSigning_len);
s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len);
//for (i=0;i < sRequest_len;i++) printf("%02x", sRequest[i]);
for (i=0;i < sRequest_len;i++) sprintf(hexsRequest+(i*2), "%02x", sRequest[i]);
StringToSign = "AWS4-HMAC-SHA256\n";
StringToSign += date3+"\n";
StringToSign += date2+"/" + endpoint + "/s3/aws4_request\n";
StringToSign += hexsRequest;
unsigned char* cscope = (unsigned char*)StringToSign.c_str();
unsigned int cscope_len = StringToSign.size();
unsigned char* md = NULL;
unsigned int md_len = 0;
char hexSig[64];
s3fs_HMAC(kSigning, kSigning_len, cscope, cscope_len, &md, &md_len);
for (i=0; i < md_len; i++) sprintf(hexSig+(i*2), "%02x", md[i]);
Signature = hexSig;
return Signature;
}
string S3fsCurl::CalcSignature(string method, string canonical_uri, string date2,
string canonical_headers, string payload_hash, string signed_headers, string date3)
{
return CalcSignatureReal(method, canonical_uri, "", date2, canonical_headers, payload_hash, signed_headers, date3);
}
#endif
// XML in BodyData has UploadId, Parse XML body for UploadId // XML in BodyData has UploadId, Parse XML body for UploadId
bool S3fsCurl::GetUploadId(string& upload_id) bool S3fsCurl::GetUploadId(string& upload_id)
{ {
@ -1852,6 +1978,7 @@ int S3fsCurl::DeleteRequest(const char* tpath)
requestHeaders = NULL; requestHeaders = NULL;
responseHeaders.clear(); responseHeaders.clear();
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: ");
@ -1861,6 +1988,30 @@ int S3fsCurl::DeleteRequest(const char* tpath)
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("DELETE", "", "", date, resource)).c_str()); CalcSignature("DELETE", "", "", date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=tpath;
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \
+ payload_hash + "\nx-amz-date:" + date3 + "\n";
string signed_headers = "host;x-amz-content-sha256;x-amz-date";
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId +
"/" + date2 + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("DELETE", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
@ -1966,6 +2117,7 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
responseHeaders.clear(); responseHeaders.clear();
// requestHeaders // requestHeaders
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: ");
@ -1984,6 +2136,31 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("HEAD", "", "", date, resource)).c_str()); CalcSignature("HEAD", "", "", date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=tpath;
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \
+ payload_hash + "\nx-amz-date:" + date3 + "\n";
string signed_headers = "host;x-amz-content-sha256;x-amz-date";
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
//DPRN("SHUNDEBUG3SIGN %s", string("Date:"+date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("HEAD", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_NOBODY, true); // HEAD curl_easy_setopt(hCurl, CURLOPT_NOBODY, true); // HEAD
@ -2081,6 +2258,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
bodydata = new BodyData(); bodydata = new BodyData();
// Make request headers // Make request headers
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
@ -2108,6 +2286,7 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
} }
} }
} }
// "x-amz-acl", rrs, sse // "x-amz-acl", rrs, sse
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str());
if(S3fsCurl::is_use_rrs){ if(S3fsCurl::is_use_rrs){
@ -2131,6 +2310,93 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("PUT", "", ContentType, date, resource)).c_str()); CalcSignature("PUT", "", ContentType, date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=tpath;
//string payload_hash = s3fs_sha256sum(fd, 0, -1);
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string ContentType;
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n";
string signed_headers = "host";
// "x-amz-acl"
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str());
canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n";
signed_headers += ";x-amz-acl";
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n";
signed_headers += ";x-amz-content-sha256";
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
canonical_headers += "x-amz-date:" + date3 + "\n";
signed_headers += ";x-amz-date";
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string key = (*iter).first;
string value = (*iter).second;
if(0 == strcasecmp(key.c_str(), "Content-Type")){
ContentType = value;
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
canonical_headers.insert(0, "content-type:" + value + "\n");
signed_headers.insert(0, "content-type;");
}else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
// not set value, but after set it.
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
canonical_headers += key + ":" + value + "\n";
signed_headers += ";" + key;
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n"));
signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key));
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
// skip this header, because this header is specified after logic.
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
}else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
// Only copy mode.
if(!AddSseKeyRequestHead(value, is_copy)){
DPRNNN("Failed to insert sse(-c) header.");
}
}
}
// rrs, sse
if(S3fsCurl::is_use_sse){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
canonical_headers += "x-amz-server-side-encryption:AES256\n";
signed_headers += ";x-amz-server-side-encryption";
}else if(S3fsCurl::IsSseCustomMode()){
string md5;
if(!AddSseKeyRequestHead(md5, false)){
DPRNNN("Failed to insert sse(-c) header.");
}
}
if(S3fsCurl::is_use_rrs){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n";
signed_headers += ";x-amz-storage-class";
}
if(is_use_ahbe){
// set additional header by ahbe conf
requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
}
//DPRN("SHUNDEBUG6PUT %s", string("Date:"+date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2191,6 +2457,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
bodydata = new BodyData(); bodydata = new BodyData();
// Make request headers // Make request headers
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
@ -2242,6 +2509,96 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("PUT", strMD5, ContentType, date, resource)).c_str()); CalcSignature("PUT", strMD5, ContentType, date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=tpath;
string payload_hash = s3fs_sha256sum(fd, 0, -1);
if(0 == strlen(payload_hash.c_str()))
payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
//string canonical_headers, signed_headers;
string ContentType;
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n";
string signed_headers = "host";
string strMD5;
// "x-amz-acl"
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str());
canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n";
signed_headers += ";x-amz-acl";
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n";
signed_headers += ";x-amz-content-sha256";
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
canonical_headers += "x-amz-date:" + date3 + "\n";
signed_headers += ";x-amz-date";
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string key = (*iter).first;
string value = (*iter).second;
if(0 == strcasecmp(key.c_str(), "Content-Type")){
ContentType = value;
requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type:" + value).c_str());
canonical_headers.insert(0, "content-type:" + value +"\n");
signed_headers.insert(0, "content-type;");
}else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
// not set value, but after set it.
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
canonical_headers += key + ":" + value + "\n";
signed_headers += ";" + key;
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
// skip this header, because this header is specified after logic.
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
// skip this header, because this header is specified after logic.
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
// skip this header, because this header is specified after logic.
}
}
if(-1 != fd && S3fsCurl::is_content_md5){
// content-md5 must be before content-type
strMD5 = s3fs_get_content_md5(fd);
// content-md5 must be before content-type
requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-md5: " + strMD5).c_str());
canonical_headers.insert(0, "content-md5:" + strMD5 +"\n");
signed_headers.insert(0, "content-md5;");
}
if(S3fsCurl::is_use_sse){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
canonical_headers += "x-amz-server-side-encryption:AES256\n";
signed_headers += ";x-amz-server-side-encryption";
}else if(S3fsCurl::IsSseCustomMode()){
string md5;
if(!AddSseKeyRequestHead(md5, false)){
DPRNNN("Failed to insert sse(-c) header.");
}
}
if(S3fsCurl::is_use_rrs){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n";
signed_headers += ";x-amz-storage-class";
}
if(is_use_ahbe){
// set additional header by ahbe conf
requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
}
//DPRN("SHUNDEBUG5PUT %s", string("Date:"+date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2290,6 +2647,7 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
requestHeaders = NULL; requestHeaders = NULL;
responseHeaders.clear(); responseHeaders.clear();
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: ");
@ -2312,6 +2670,41 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("GET", "", "", date, resource)).c_str()); CalcSignature("GET", "", "", date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=tpath;
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \
+ payload_hash + "\nx-amz-date:" + date3 + "\n";
string signed_headers = "host;x-amz-content-sha256;x-amz-date";
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
//DPRN("SHUNDEBUGxSIGN %s", string("Date:"+date3).c_str());
if(-1 != start && -1 != size){
string range = "Range: bytes=";
range += str(start);
range += "-";
range += str(start + size - 1);
requestHeaders = curl_slist_sort_insert(requestHeaders, range.c_str());
}
//DPRN("SHUNDEBUGxSIGN");
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("GET", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2378,6 +2771,7 @@ int S3fsCurl::CheckBucket(void)
responseHeaders.clear(); responseHeaders.clear();
bodydata = new BodyData(); bodydata = new BodyData();
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
@ -2387,6 +2781,37 @@ int S3fsCurl::CheckBucket(void)
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("GET", "", "", date, resource)).c_str()); CalcSignature("GET", "", "", date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \
+ payload_hash + "\nx-amz-date:" + date3 + "\n";
string signed_headers = "host;x-amz-content-sha256;x-amz-date";
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
//DPRN("SHUNDEBUG1SIGN %s", string("Date:"+date3).c_str());
// step 1 method : PARAM1: 'GET'
// step 2 c..uri : fixed: '/'
// step 3 query string: ''
// step 4 c..headers : PARAM3: 3
// step 5 s..headers: PARAM5: 3
// step 6 payload_hash : PARAM4: emptyhash
// step 7 c..request in calcsignature 1+...+6
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("GET", "", date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)bodydata);
@ -2429,6 +2854,7 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
responseHeaders.clear(); responseHeaders.clear();
bodydata = new BodyData(); bodydata = new BodyData();
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: "); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: ");
@ -2439,6 +2865,32 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("GET", "", "", date, (resource + "/"))).c_str()); CalcSignature("GET", "", "", date, (resource + "/"))).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=query;
//DPRN("SHUNDEBUG2SIGN %s\t%s", string("CURL:"+ canonical_uri).c_str(), tpath);
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string signed_headers = "host;x-amz-content-sha256;x-amz-date";
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \
+ payload_hash + "\nx-amz-date:" + date3 + "\n";
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
//DPRN("SHUNDEBUG2SIGN %s", string("Date:"+date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("GET", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2474,6 +2926,7 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
string turl; string turl;
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
#ifdef SIGV3
turl += "?uploads"; turl += "?uploads";
resource += "?uploads"; resource += "?uploads";
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
@ -2531,6 +2984,103 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("POST", "", contype, date, resource)).c_str()); CalcSignature("POST", "", contype, date, resource)).c_str());
} }
#else
string query_string = "uploads=";
turl += "?"+query_string;
resource += "?"+query_string;
url = prepare_url(turl.c_str());
path = tpath;
requestHeaders = NULL;
responseHeaders.clear();
bodydata = new BodyData();
DPRN("SHUNDEBUG9POST %s", tpath); //string("Date:"+date3).c_str());
// to be done: what's the payload_hash value?
string payload_hash = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=tpath;
string contype = S3fsCurl::LookupMimeType(string(tpath));
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n";
string signed_headers = "host";
// "x-amz-acl"
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str());
canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n";
signed_headers += ";x-amz-acl";
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n";
signed_headers += ";x-amz-content-sha256";
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: ");
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length: ");
requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type: " + contype).c_str());
canonical_headers.insert(0, "content-type:" + contype +"\n");
signed_headers.insert(0, "content-type;");
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
canonical_headers += "x-amz-date:" + date3 + "\n";
signed_headers += ";x-amz-date";
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string key = (*iter).first;
string value = (*iter).second;
if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
// not set value, but after set it.
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
canonical_headers += key + ":" + value + "\n";
signed_headers += ";" + key;
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
// skip this header, because this header is specified after logic.
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
}else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
// Only copy mode.
if(!AddSseKeyRequestHead(value, is_copy)){
DPRNNN("Failed to insert sse(-c) header.");
}
}
}
// rrs, sse
if(S3fsCurl::is_use_sse){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
canonical_headers += "x-amz-server-side-encryption:AES256\n";
signed_headers += ";x-amz-server-side-encryption";
}else if(S3fsCurl::IsSseCustomMode()){
string md5;
if(!AddSseKeyRequestHead(md5, false)){
DPRNNN("Failed to insert sse(-c) header.");
}
}
if(S3fsCurl::is_use_rrs){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n";
signed_headers += ";x-amz-storage-class";
}
if(is_use_ahbe){
// set additional header by ahbe conf
requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
}
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignatureReal("POST", canonical_uri, query_string, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2598,6 +3148,7 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id,
string turl; string turl;
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
#ifdef SIGV3
turl += "?uploadId=" + upload_id; turl += "?uploadId=" + upload_id;
resource += "?uploadId=" + upload_id; resource += "?uploadId=" + upload_id;
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
@ -2617,6 +3168,67 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id,
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("POST", "", "", date, resource)).c_str()); CalcSignature("POST", "", "", date, resource)).c_str());
} }
#else
string query_string;
query_string = "uploadId=" + upload_id;
turl += "?" + query_string;
resource += "?" + query_string;
url = prepare_url(turl.c_str());
path = tpath;
requestHeaders = NULL;
responseHeaders.clear();
bodydata = new BodyData();
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
DPRN("SHUNDEBUG9POST %s", tpath); //string("Date:"+date3).c_str());
// to be done: what's the payload_hash value?
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
unsigned char * cRequest = (unsigned char *)postContent.c_str(), *sRequest;
unsigned int cRequest_len= postContent.size();
unsigned int sRequest_len = 0;
char hexsRequest[64];
unsigned int i;
s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len);
for (i=0;i < sRequest_len;i++) sprintf(hexsRequest+(i*2), "%02x", sRequest[i]);
payload_hash.assign(hexsRequest, hexsRequest+sRequest_len*2);
string canonical_uri = "";
canonical_uri +=tpath;
string contype = S3fsCurl::LookupMimeType(string(tpath));
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n";
string signed_headers = "host";
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept:");
requestHeaders = curl_slist_sort_insert(requestHeaders, string("content-type: " + contype).c_str());
canonical_headers.insert(0, "content-type:" + contype +"\n");
signed_headers.insert(0, "content-type;");
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n";
signed_headers += ";x-amz-content-sha256";
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
canonical_headers += "x-amz-date:" + date3 + "\n";
signed_headers += ";x-amz-date";
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignatureReal("POST", canonical_uri, query_string, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2658,6 +3270,7 @@ int S3fsCurl::MultipartListRequest(string& body)
responseHeaders.clear(); responseHeaders.clear();
bodydata = new BodyData(); bodydata = new BodyData();
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: "); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept: ");
@ -2668,6 +3281,29 @@ int S3fsCurl::MultipartListRequest(string& body)
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("GET", "", "", date, resource)).c_str()); CalcSignature("GET", "", "", date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \
+ payload_hash + "\nx-amz-date:" + date3 + "\n";
string signed_headers = "host;x-amz-content-sha256;x-amz-date";
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
//DPRN("SHUNDEBUG4SIGN %s", string("Date:"+date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("GET", "", date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2710,6 +3346,7 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, string& upload_id)
requestHeaders = NULL; requestHeaders = NULL;
responseHeaders.clear(); responseHeaders.clear();
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
if(!S3fsCurl::IsPublicBucket()){ if(!S3fsCurl::IsPublicBucket()){
@ -2718,6 +3355,30 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, string& upload_id)
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("DELETE", "", "", date, resource)).c_str()); CalcSignature("DELETE", "", "", date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=tpath;
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\nx-amz-content-sha256:" \
+ payload_hash + "\nx-amz-date:" + date3 + "\n";
string signed_headers = "host;x-amz-content-sha256;x-amz-date";
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId +
"/" + date2 + "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("DELETE", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
@ -2764,6 +3425,7 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string&
} }
// make request // make request
#ifdef SIGV3
string urlargs = "?partNumber=" + IntToStr(part_num) + "&uploadId=" + upload_id; string urlargs = "?partNumber=" + IntToStr(part_num) + "&uploadId=" + upload_id;
string resource; string resource;
string turl; string turl;
@ -2788,6 +3450,55 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, string&
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("PUT", "", "", date, resource)).c_str()); CalcSignature("PUT", "", "", date, resource)).c_str());
} }
#else
string request_uri = "partNumber=" + IntToStr(part_num) + "&uploadId=" + upload_id;
string urlargs = "?" + request_uri;
string resource;
string turl;
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
resource += urlargs;
turl += urlargs;
url = prepare_url(turl.c_str());
path = tpath;
requestHeaders = NULL;
responseHeaders.clear();
bodydata = new BodyData();
headdata = new BodyData();
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=tpath;
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string ContentType;
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n";
string signed_headers = "host";
//payload_hash="AWS4-HMAC-SHA256-PAYLOAD";
payload_hash = s3fs_sha256sum(partdata.fd, partdata.startpos, partdata.size);
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n";
signed_headers += ";x-amz-content-sha256";
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
canonical_headers += "x-amz-date:" + date3 + "\n";
signed_headers += ";x-amz-date";
DPRN("SHUNDEBUG7PUT %s", string("Date:"+date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignatureReal("PUT", canonical_uri, request_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2861,6 +3572,7 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
headdata = new BodyData(); headdata = new BodyData();
// Make request headers // Make request headers
#ifdef SIGV3
string date = get_date(); string date = get_date();
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
@ -2884,6 +3596,68 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
string("Authorization: AWS " + AWSAccessKeyId + ":" + string("Authorization: AWS " + AWSAccessKeyId + ":" +
CalcSignature("PUT", "", ContentType, date, resource)).c_str()); CalcSignature("PUT", "", ContentType, date, resource)).c_str());
} }
#else
string date = get_date();
string date2 = get_date2();
string date3 = get_date3();
string canonical_uri = "";
canonical_uri +=path;
string payload_hash = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
string ContentType;
requestHeaders = curl_slist_sort_insert(requestHeaders, string("host:" + bucket + "." + "s3.amazonaws.com").c_str());
string canonical_headers = "host:" + bucket + "." + "s3.amazonaws.com" + "\n";
string signed_headers = "host";
/* not need
// "x-amz-acl"
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-acl:" + S3fsCurl::default_acl).c_str());
canonical_headers += "x-amz-acl:" + S3fsCurl::default_acl + "\n";
signed_headers += ";x-amz-acl";
*/
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-content-sha256:" + payload_hash).c_str());
canonical_headers += "x-amz-content-sha256:" + payload_hash + "\n";
signed_headers += ";x-amz-content-sha256";
requestHeaders = curl_slist_sort_insert(requestHeaders,
string("x-amz-date:" + date3).c_str());
canonical_headers += "x-amz-date:" + date3 + "\n";
signed_headers += ";x-amz-date";
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string key = (*iter).first;
string value = (*iter).second;
if(0 == strcasecmp(key.c_str(), "Content-Type")){
ContentType = value;
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
canonical_headers.insert(0, "content-type:" + value + "\n");
signed_headers.insert(0, "content-type;");
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n"));
signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key));
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source-range")){
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
canonical_headers.insert(canonical_headers.find("x-amz-date:" + date3 + "\n"), string(key + ":" + value + "\n"));
signed_headers.insert(signed_headers.find(";x-amz-date"), string(";" + key));
}
// NOTICE: x-amz-acl, x-amz-server-side-encryption is not set!
}
if(S3fsCurl::is_use_rrs){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
canonical_headers += "x-amz-storage-class:REDUCED_REDUNDANCY\n";
signed_headers += ";x-amz-storage-class";
}
DPRN("SHUNDEBUG8PUT %s", string("Date:"+date3).c_str());
if(!S3fsCurl::IsPublicBucket()){
requestHeaders = curl_slist_sort_insert(
requestHeaders,
string("Authorization: AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + date2
+ "/" + endpoint + "/s3/aws4_request, SignedHeaders=" + signed_headers + ", Signature=" +
CalcSignature("PUT", canonical_uri, date2, canonical_headers, payload_hash, signed_headers, date3)).c_str());
}
#endif
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());

View File

@ -244,7 +244,15 @@ class S3fsCurl
bool ResetHandle(void); bool ResetHandle(void);
bool RemakeHandle(void); bool RemakeHandle(void);
bool ClearInternalData(void); bool ClearInternalData(void);
#ifdef SIGV3
std::string CalcSignature(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource); std::string CalcSignature(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource);
#else
std::string CalcSignaturev2(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource);
std::string CalcSignature(std::string method, std::string canonical_uri, std::string date2, std::string cononical_headers,
std::string payload_hash, std::string signed_headers, std::string date3);
std::string CalcSignatureReal(std::string method, std::string canonical_uri, std::string query_string, std::string date2, std::string cononical_headers,
std::string payload_hash, std::string signed_headers, std::string date3);
#endif
bool GetUploadId(std::string& upload_id); bool GetUploadId(std::string& upload_id);
int GetIAMCredentials(void); int GetIAMCredentials(void);

View File

@ -32,6 +32,9 @@
#include <openssl/evp.h> #include <openssl/evp.h>
#include <openssl/hmac.h> #include <openssl/hmac.h>
#include <openssl/md5.h> #include <openssl/md5.h>
#ifndef SIGV3
#include <openssl/sha.h>
#endif
#include <openssl/crypto.h> #include <openssl/crypto.h>
#include <openssl/err.h> #include <openssl/err.h>
#include <string> #include <string>
@ -191,7 +194,11 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){ if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){
return false; return false;
} }
#ifdef SIGV3
HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen); HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen);
#else
HMAC(EVP_sha256(), key, keylen, data, datalen, *digest, digestlen);
#endif
return true; return true;
} }
@ -255,6 +262,85 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
return result; return result;
} }
#ifndef SIGV3
//-------------------------------------------------------------------
// Utility Function for SHA256
//-------------------------------------------------------------------
size_t get_sha256_digest_length(void)
{
return SHA256_DIGEST_LENGTH;
}
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
{
(*digestlen) = EVP_MAX_MD_SIZE * sizeof(unsigned char);
if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){
return false;
}
const EVP_MD *md = EVP_get_digestbyname("sha256");
EVP_MD_CTX *mdctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(mdctx, md, NULL);
EVP_DigestUpdate(mdctx, data, datalen);
EVP_DigestFinal_ex(mdctx, *digest, digestlen);
EVP_MD_CTX_destroy(mdctx);
return true;
}
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
{
const EVP_MD *md = EVP_get_digestbyname("sha256");
EVP_MD_CTX *sha256ctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(sha256ctx, md, NULL);
char buf[512];
ssize_t bytes;
unsigned char* result;
if(-1 == size) {
struct stat st;
if(-1 == fstat(fd, &st)){
return NULL;
}
size = static_cast<ssize_t>(st.st_size);
}
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512);
for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes);
if(0 == bytes){
// end of file
break;
}else if(-1 == bytes){
// error
DPRNNN("file read error(%d)", errno);
return NULL;
}
EVP_DigestUpdate(sha256ctx, buf, bytes);
memset(buf, 0, 512);
}
if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
return NULL;
}
EVP_DigestFinal_ex(sha256ctx, result, NULL);
EVP_MD_CTX_destroy(sha256ctx);
if(-1 == lseek(fd, start, SEEK_SET)){
free(result);
return NULL;
}
return result;
}
#endif
/* /*
* Local variables: * Local variables:
* tab-width: 4 * tab-width: 4

View File

@ -88,6 +88,9 @@ std::string program_name;
std::string service_path = "/"; std::string service_path = "/";
std::string host = "http://s3.amazonaws.com"; std::string host = "http://s3.amazonaws.com";
std::string bucket = ""; std::string bucket = "";
#ifndef SIGV3
std::string endpoint = "us-east-1";
#endif
//------------------------------------------------------------------- //-------------------------------------------------------------------
// Static valiables // Static valiables
@ -3759,6 +3762,12 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
} }
return 0; return 0;
} }
#ifndef SIGV3
if(0 == STR2NCMP(arg, "endpoint=")){
endpoint = strchr(arg, '=') + sizeof(char);
return 0;
}
#endif
if(0 == strcmp(arg, "use_path_request_style")){ if(0 == strcmp(arg, "use_path_request_style")){
pathrequeststyle = true; pathrequeststyle = true;
return 0; return 0;

View File

@ -29,6 +29,9 @@
char* s3fs_base64(unsigned char* input, size_t length); char* s3fs_base64(unsigned char* input, size_t length);
std::string s3fs_get_content_md5(int fd); std::string s3fs_get_content_md5(int fd);
std::string s3fs_md5sum(int fd, off_t start, ssize_t size); std::string s3fs_md5sum(int fd, off_t start, ssize_t size);
#ifndef SIGV3
std::string s3fs_sha256sum(int fd, off_t start, ssize_t size);
#endif
// //
// in xxxxxx_auth.cpp // in xxxxxx_auth.cpp
@ -41,6 +44,11 @@ bool s3fs_destroy_crypt_mutex(void);
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen); bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen);
size_t get_md5_digest_length(void); size_t get_md5_digest_length(void);
unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size); unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size);
#ifndef SIGV3
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen);
size_t get_sha256_digest_length(void);
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size);
#endif
#endif // S3FS_AUTH_H_ #endif // S3FS_AUTH_H_

View File

@ -975,6 +975,11 @@ void show_help (void)
"\n" "\n"
" url (default=\"http://s3.amazonaws.com\")\n" " url (default=\"http://s3.amazonaws.com\")\n"
" - sets the url to use to access amazon s3\n" " - sets the url to use to access amazon s3\n"
#ifndef SIGV3
"\n"
" endpoint (default=\"us-east-1\")\n"
" - sets the endpoint to use\n"
#endif
"\n" "\n"
" nomultipart (disable multipart uploads)\n" " nomultipart (disable multipart uploads)\n"
"\n" "\n"

View File

@ -125,6 +125,12 @@ string urlEncode(const string &s)
for (unsigned i = 0; i < s.length(); ++i) { for (unsigned i = 0; i < s.length(); ++i) {
if (s[i] == '/') { // Note- special case for fuse paths... if (s[i] == '/') { // Note- special case for fuse paths...
result += s[i]; result += s[i];
#ifndef SIGV3
}else if (s[i] == '=') { // Note- special case for s3...
result += s[i];
}else if (s[i] == '&') { // Note- special case for s3...
result += s[i];
#endif
} else if (isalnum(s[i])) { } else if (isalnum(s[i])) {
result += s[i]; result += s[i];
} else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') { } else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') {
@ -143,6 +149,39 @@ string urlEncode(const string &s)
return result; return result;
} }
#ifndef SIGV3
/**
* urlEncode a fuse path,
* taking into special consideration "/",
* otherwise regular urlEncode.
*/
string urlEncode2(const string &s)
{
string result;
for (unsigned i = 0; i < s.length(); ++i) {
if (s[i] == '=') { // Note- special case for fuse paths...
result += s[i];
}else if (s[i] == '&') { // Note- special case for s3...
result += s[i];
} else if (isalnum(s[i])) {
result += s[i];
} else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') {
result += s[i];
} else if (s[i] == ' ') {
result += '%';
result += '2';
result += '0';
} else {
result += "%";
result += hexAlphabet[static_cast<unsigned char>(s[i]) / 16];
result += hexAlphabet[static_cast<unsigned char>(s[i]) % 16];
}
}
return result;
}
#endif
// //
// ex. target="http://......?keyword=value&..." // ex. target="http://......?keyword=value&..."
// //
@ -181,6 +220,25 @@ string get_date()
return buf; return buf;
} }
#ifndef SIGV3
string get_date2()
{
char buf[100];
time_t t = time(NULL);
strftime(buf, sizeof(buf), "%Y%m%d", gmtime(&t));
return buf;
}
string get_date3()
{
char buf[100];
time_t t = time(NULL);
strftime(buf, sizeof(buf), "%Y%m%dT%H%M%SZ", gmtime(&t));
return buf;
}
#endif
/* /*
* Local variables: * Local variables:
* tab-width: 4 * tab-width: 4

View File

@ -47,7 +47,14 @@ std::string trim(const std::string &s, const std::string &t = SPACES);
std::string lower(std::string s); std::string lower(std::string s);
std::string IntToStr(int); std::string IntToStr(int);
std::string get_date(); std::string get_date();
#ifndef SIGV3
std::string get_date2();
std::string get_date3();
#endif
std::string urlEncode(const std::string &s); std::string urlEncode(const std::string &s);
#ifndef SIGV3
std::string urlEncode2(const std::string &s);
#endif
bool get_keyword_value(std::string& target, const char* keyword, std::string& value); bool get_keyword_value(std::string& target, const char* keyword, std::string& value);
#endif // S3FS_STRING_UTIL_H_ #endif // S3FS_STRING_UTIL_H_