octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2025-01-11 00:21:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

auth headers insertion refactoring

Browse Source
This commit is contained in:
Or Ozeri 2017-10-26 17:21:48 +03:00
parent ab89b4cd4a
commit 384b4cbafa
2 changed files with 152 additions and 248 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

371
src/curl.cpp
View File

@ -1564,7 +1564,7 @@ S3fsCurl::S3fsCurl(bool ahbe) :
hCurl(NULL), type(REQTYPE_UNSET), path(""), base_path(""), saved_path(""), url(""), requestHeaders(NULL), hCurl(NULL), type(REQTYPE_UNSET), path(""), base_path(""), saved_path(""), url(""), requestHeaders(NULL),
bodydata(NULL), headdata(NULL), LastResponseCode(-1), postdata(NULL), postdata_remaining(0), is_use_ahbe(ahbe), bodydata(NULL), headdata(NULL), LastResponseCode(-1), postdata(NULL), postdata_remaining(0), is_use_ahbe(ahbe),
retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0), retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0),
b_ssekey_pos(-1), b_ssevalue(""), b_ssetype(SSE_DISABLE) b_ssekey_pos(-1), b_ssevalue(""), b_ssetype(SSE_DISABLE), op(""), query_string("")
{ {
} }
@ -1679,8 +1679,10 @@ bool S3fsCurl::ClearInternalData(void)
type = REQTYPE_UNSET; type = REQTYPE_UNSET;
path = ""; path = "";
base_path = ""; base_path = "";
saved_path= ""; saved_path = "";
url = ""; url = "";
op = "";
query_string= "";
if(requestHeaders){ if(requestHeaders){
curl_slist_free_all(requestHeaders); curl_slist_free_all(requestHeaders);
requestHeaders = NULL; requestHeaders = NULL;
@ -2263,15 +2265,45 @@ bool S3fsCurl::GetUploadId(string& upload_id)
return result; return result;
} }
void S3fsCurl::insertV4Headers(const string &op, const string &path, const string &query_string, const string &payload_hash) void S3fsCurl::insertV4Headers()
{ {
S3FS_PRN_INFO3("computing signature [%s] [%s] [%s] [%s]", op.c_str(), path.c_str(), query_string.c_str(), payload_hash.c_str()); string server_path = type == REQTYPE_LISTBUCKET ? "/" : path;
string payload_hash;
switch (type) {
case REQTYPE_PUT:
payload_hash = s3fs_sha256sum(b_infile == NULL ? -1 : fileno(b_infile), 0, -1);
break;
case REQTYPE_COMPLETEMULTIPOST:
{
unsigned int cRequest_len = strlen(reinterpret_cast<const char *>(b_postdata));
unsigned char* sRequest = NULL;
unsigned int sRequest_len = 0;
char hexsRequest[64 + 1];
unsigned int cnt;
s3fs_sha256(b_postdata, cRequest_len, &sRequest, &sRequest_len);
for(cnt = 0; cnt < sRequest_len; cnt++){
sprintf(&hexsRequest[cnt * 2], "%02x", sRequest[cnt]);
}
free(sRequest);
payload_hash.assign(hexsRequest, &hexsRequest[sRequest_len * 2]);
break;
}
case REQTYPE_UPLOADMULTIPOST:
payload_hash = s3fs_sha256sum(partdata.fd, partdata.startpos, partdata.size);
break;
default:
break;
}
S3FS_PRN_INFO3("computing signature [%s] [%s] [%s] [%s]", op.c_str(), server_path.c_str(), query_string.c_str(), payload_hash.c_str());
string strdate; string strdate;
string date8601; string date8601;
get_date_sigv3(strdate, date8601); get_date_sigv3(strdate, date8601);
string contentSHA256 = payload_hash.empty() ? empty_payload_hash : payload_hash; string contentSHA256 = payload_hash.empty() ? empty_payload_hash : payload_hash;
const std::string realpath = pathrequeststyle ? "/" + bucket + path : path; const std::string realpath = pathrequeststyle ? "/" + bucket + server_path : server_path;
//string canonical_headers, signed_headers; //string canonical_headers, signed_headers;
requestHeaders = curl_slist_sort_insert(requestHeaders, "host", get_bucket_host().c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "host", get_bucket_host().c_str());
@ -2279,13 +2311,44 @@ void S3fsCurl::insertV4Headers(const string &op, const string &path, const strin
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-date", date8601.c_str());
if(!S3fsCurl::IsPublicBucket()){ if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignature(op, realpath, query_string, strdate, contentSHA256, date8601); string Signature = CalcSignature(op, realpath, query_string + (type == REQTYPE_PREMULTIPOST ? "=" : ""), strdate, contentSHA256, date8601);
string auth = "AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint + string auth = "AWS4-HMAC-SHA256 Credential=" + AWSAccessKeyId + "/" + strdate + "/" + endpoint +
"/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature; "/s3/aws4_request, SignedHeaders=" + get_sorted_header_keys(requestHeaders) + ", Signature=" + Signature;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", auth.c_str());
} }
} }
void S3fsCurl::insertV2Headers()
{
string resource;
string turl;
string server_path = type == REQTYPE_LISTBUCKET ? "/" : path;
MakeUrlResource(server_path.c_str(), resource, turl);
if(!query_string.empty() && type != REQTYPE_LISTBUCKET){
resource += "?" + query_string;
}
string date = get_date_rfc850();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
if(op != "PUT" && op != "POST"){
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL);
}
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2(op, get_header_value(requestHeaders, "Content-MD5"), get_header_value(requestHeaders, "Content-Type"), date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}
void S3fsCurl::insertAuthHeaders()
{
if(!S3fsCurl::is_sigv4){
insertV2Headers();
}else{
insertV4Headers();
}
}
int S3fsCurl::DeleteRequest(const char* tpath) int S3fsCurl::DeleteRequest(const char* tpath)
{ {
S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath)); S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
@ -2305,26 +2368,15 @@ int S3fsCurl::DeleteRequest(const char* tpath)
requestHeaders = NULL; requestHeaders = NULL;
responseHeaders.clear(); responseHeaders.clear();
if(!S3fsCurl::is_sigv4){ op = "DELETE";
string date = get_date_rfc850(); type = REQTYPE_DELETE;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); insertAuthHeaders();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL);
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("DELETE", "", "", date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
insertV4Headers("DELETE", path, "", "");
}
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_DELETE;
return RequestPerform(); return RequestPerform();
} }
@ -2481,19 +2533,9 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
} }
b_ssekey_pos = ssekey_pos; b_ssekey_pos = ssekey_pos;
if(!S3fsCurl::is_sigv4){ op = "HEAD";
string date = get_date_rfc850(); type = REQTYPE_HEAD;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); insertAuthHeaders();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL);
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("HEAD", "", "", date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
insertV4Headers("HEAD", path, "", "");
}
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_NOBODY, true); // HEAD curl_easy_setopt(hCurl, CURLOPT_NOBODY, true); // HEAD
@ -2505,8 +2547,6 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback); curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_HEAD;
return true; return true;
} }
@ -2578,12 +2618,10 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
bodydata = new BodyData(); bodydata = new BodyData();
// Make request headers // Make request headers
string ContentType;
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string key = lower(iter->first); string key = lower(iter->first);
string value = iter->second; string value = iter->second;
if(key == "content-type"){ if(key == "content-type"){
ContentType = value;
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
}else if(key.substr(0, 9) == "x-amz-acl"){ }else if(key.substr(0, 9) == "x-amz-acl"){
// not set value, but after set it. // not set value, but after set it.
@ -2632,18 +2670,9 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
} }
if(!S3fsCurl::is_sigv4){ op = "PUT";
string date = get_date_rfc850(); type = REQTYPE_PUTHEAD;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); insertAuthHeaders();
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("PUT", "", ContentType, date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
insertV4Headers("PUT", path, "", "");
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2654,8 +2683,6 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_PUTHEAD;
S3FS_PRN_INFO3("copying... [path=%s]", tpath); S3FS_PRN_INFO3("copying... [path=%s]", tpath);
int result = RequestPerform(); int result = RequestPerform();
@ -2734,12 +2761,10 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", strMD5.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", strMD5.c_str());
} }
string ContentType;
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string key = lower(iter->first); string key = lower(iter->first);
string value = iter->second; string value = iter->second;
if(key == "content-type"){ if(key == "content-type"){
ContentType = value;
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
}else if(key.substr(0, 9) == "x-amz-acl"){ }else if(key.substr(0, 9) == "x-amz-acl"){
// not set value, but after set it. // not set value, but after set it.
@ -2772,19 +2797,9 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
} }
if(!S3fsCurl::is_sigv4){ op = "PUT";
string date = get_date_rfc850(); type = REQTYPE_PUT;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); insertAuthHeaders();
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("PUT", strMD5, ContentType, date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
string payload_hash = s3fs_sha256sum(fd, 0, -1);
insertV4Headers("PUT", path, "", payload_hash);
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2800,8 +2815,6 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
} }
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_PUT;
S3FS_PRN_INFO3("uploading... [path=%s][fd=%d][size=%jd]", tpath, fd, (intmax_t)(-1 != fd ? st.st_size : 0)); S3FS_PRN_INFO3("uploading... [path=%s][fd=%d][size=%jd]", tpath, fd, (intmax_t)(-1 != fd ? st.st_size : 0));
int result = RequestPerform(); int result = RequestPerform();
@ -2845,19 +2858,10 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
if(!AddSseRequestHead(ssetype, ssevalue, true, false)){ if(!AddSseRequestHead(ssetype, ssevalue, true, false)){
S3FS_PRN_WARN("Failed to set SSE header, but continue..."); S3FS_PRN_WARN("Failed to set SSE header, but continue...");
} }
if(!S3fsCurl::is_sigv4){
string date = get_date_rfc850();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL);
if(!S3fsCurl::IsPublicBucket()){ op = "GET";
string Signature = CalcSignatureV2("GET", "", "", date, resource); type = REQTYPE_GET;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); insertAuthHeaders();
}
}else{
insertV4Headers("GET", path, "", "");
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2878,8 +2882,6 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
b_ssevalue = ssevalue; b_ssevalue = ssevalue;
b_ssekey_pos = -1; // not use this value for get object. b_ssekey_pos = -1; // not use this value for get object.
type = REQTYPE_GET;
return 0; return 0;
} }
@ -2927,18 +2929,9 @@ int S3fsCurl::CheckBucket(void)
responseHeaders.clear(); responseHeaders.clear();
bodydata = new BodyData(); bodydata = new BodyData();
if(!S3fsCurl::is_sigv4){ op = "GET";
string date = get_date_rfc850(); type = REQTYPE_CHKBUCKET;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); insertAuthHeaders();
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("GET", "", "", date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
insertV4Headers("GET", path, "", "");
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -2947,8 +2940,6 @@ int S3fsCurl::CheckBucket(void)
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_CHKBUCKET;
int result = RequestPerform(); int result = RequestPerform();
if (result != 0) { if (result != 0) {
S3FS_PRN_ERR("Check bucket failed, S3 response: %s", (bodydata ? bodydata->str() : "")); S3FS_PRN_ERR("Check bucket failed, S3 response: %s", (bodydata ? bodydata->str() : ""));
@ -2972,6 +2963,7 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
if(query){ if(query){
turl += "?"; turl += "?";
turl += query; turl += query;
query_string = query;
} }
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
@ -2980,19 +2972,9 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
responseHeaders.clear(); responseHeaders.clear();
bodydata = new BodyData(); bodydata = new BodyData();
if(!S3fsCurl::is_sigv4){ op = "GET";
string date = get_date_rfc850(); type = REQTYPE_LISTBUCKET;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); insertAuthHeaders();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", NULL);
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("GET", "", "", date, (resource + "/"));
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
insertV4Headers("GET", "/", query ? query : "", "");
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -3001,8 +2983,6 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_LISTBUCKET;
return RequestPerform(); return RequestPerform();
} }
@ -3029,9 +3009,8 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
string turl; string turl;
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
string query_string = "uploads"; query_string = "uploads";
turl += "?" + query_string; turl += "?" + query_string;
resource += "?" + query_string;
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
path = get_realpath(tpath); path = get_realpath(tpath);
requestHeaders = NULL; requestHeaders = NULL;
@ -3087,24 +3066,13 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath); requestHeaders = AdditionalHeader::get()->AddHeader(requestHeaders, tpath);
} }
if(!S3fsCurl::is_sigv4){
string date = get_date_rfc850();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length", NULL); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length", NULL);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
if(!S3fsCurl::IsPublicBucket()){ op = "POST";
string Signature = CalcSignatureV2("POST", "", contype, date, resource); type = REQTYPE_PREMULTIPOST;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); insertAuthHeaders();
}
}else{
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Length", NULL);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
insertV4Headers("POST", path, query_string + "=", ""); // NOTICE : appends "=" to query_string
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -3115,8 +3083,6 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_PREMULTIPOST;
// request // request
int result; int result;
if(0 != (result = RequestPerform())){ if(0 != (result = RequestPerform())){
@ -3173,9 +3139,8 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id,
string turl; string turl;
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
string query_string = "uploadId=" + upload_id; query_string = "uploadId=" + upload_id;
turl += "?" + query_string; turl += "?" + query_string;
resource += "?" + query_string;
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
path = get_realpath(tpath); path = get_realpath(tpath);
requestHeaders = NULL; requestHeaders = NULL;
@ -3183,39 +3148,12 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id,
responseHeaders.clear(); responseHeaders.clear();
string contype = S3fsCurl::LookupMimeType(string(tpath)); string contype = S3fsCurl::LookupMimeType(string(tpath));
if(!S3fsCurl::is_sigv4){
string date = get_date_rfc850();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
if(!S3fsCurl::IsPublicBucket()){ op = "POST";
string Signature = CalcSignatureV2("POST", "", contype, date, resource); type = REQTYPE_COMPLETEMULTIPOST;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); insertAuthHeaders();
}
}else{
string payload_hash;
const unsigned char* cRequest = reinterpret_cast<const unsigned char*>(postContent.c_str());
unsigned int cRequest_len = postContent.size();
unsigned char* sRequest = NULL;
unsigned int sRequest_len = 0;
char hexsRequest[64 + 1];
unsigned int cnt;
s3fs_sha256(cRequest, cRequest_len, &sRequest, &sRequest_len);
for(cnt = 0; cnt < sRequest_len; cnt++){
sprintf(&hexsRequest[cnt * 2], "%02x", sRequest[cnt]);
}
free(sRequest);
payload_hash.assign(hexsRequest, &hexsRequest[sRequest_len * 2]);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", get_date_rfc850().c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
insertV4Headers("POST", path, query_string, payload_hash);
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -3228,8 +3166,6 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id,
curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback); curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::ReadCallback);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_COMPLETEMULTIPOST;
// request // request
int result = RequestPerform(); int result = RequestPerform();
delete bodydata; delete bodydata;
@ -3252,25 +3188,16 @@ int S3fsCurl::MultipartListRequest(string& body)
MakeUrlResource(path.c_str(), resource, turl); MakeUrlResource(path.c_str(), resource, turl);
turl += "?uploads"; turl += "?uploads";
resource += "?uploads";
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
requestHeaders = NULL; requestHeaders = NULL;
responseHeaders.clear(); responseHeaders.clear();
bodydata = new BodyData(); bodydata = new BodyData();
if(!S3fsCurl::is_sigv4){
string date = get_date_rfc850();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
if(!S3fsCurl::IsPublicBucket()){ op = "GET";
string Signature = CalcSignatureV2("GET", "", "", date, resource); type = REQTYPE_MULTILIST;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str()); insertAuthHeaders();
}
}else{
insertV4Headers("GET", path, "", "");
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -3279,8 +3206,6 @@ int S3fsCurl::MultipartListRequest(string& body)
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_MULTILIST;
int result; int result;
if(0 == (result = RequestPerform()) && 0 < bodydata->size()){ if(0 == (result = RequestPerform()) && 0 < bodydata->size()){
body = bodydata->str(); body = bodydata->str();
@ -3308,32 +3233,20 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, string& upload_id)
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
turl += "?uploadId=" + upload_id; turl += "?uploadId=" + upload_id;
resource += "?uploadId=" + upload_id;
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
path = get_realpath(tpath); path = get_realpath(tpath);
requestHeaders = NULL; requestHeaders = NULL;
responseHeaders.clear(); responseHeaders.clear();
if(!S3fsCurl::is_sigv4){ op = "DELETE";
string date = get_date_rfc850(); type = REQTYPE_ABORTMULTIUPLOAD;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); insertAuthHeaders();
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("DELETE", "", "", date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
insertV4Headers("DELETE", path, "", "");
}
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_ABORTMULTIUPLOAD;
return RequestPerform(); return RequestPerform();
} }
@ -3361,7 +3274,6 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
} }
// make md5 and file pointer // make md5 and file pointer
std::string md5base64;
if(S3fsCurl::is_content_md5){ if(S3fsCurl::is_content_md5){
unsigned char *md5raw = s3fs_md5hexsum(partdata.fd, partdata.startpos, partdata.size); unsigned char *md5raw = s3fs_md5hexsum(partdata.fd, partdata.startpos, partdata.size);
if(md5raw == NULL){ if(md5raw == NULL){
@ -3370,7 +3282,7 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
} }
partdata.etag = s3fs_hex(md5raw, get_md5_digest_length()); partdata.etag = s3fs_hex(md5raw, get_md5_digest_length());
char* md5base64p = s3fs_base64(md5raw, get_md5_digest_length()); char* md5base64p = s3fs_base64(md5raw, get_md5_digest_length());
md5base64 = md5base64p; requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", md5base64p);
free(md5base64p); free(md5base64p);
free(md5raw); free(md5raw);
} }
@ -3381,13 +3293,12 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
} }
// make request // make request
string request_uri = "partNumber=" + str(part_num) + "&uploadId=" + upload_id; query_string = "partNumber=" + str(part_num) + "&uploadId=" + upload_id;
string urlargs = "?" + request_uri; string urlargs = "?" + query_string;
string resource; string resource;
string turl; string turl;
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl); MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
resource += urlargs;
turl += urlargs; turl += urlargs;
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
path = get_realpath(tpath); path = get_realpath(tpath);
@ -3404,26 +3315,11 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
} }
} }
if(!S3fsCurl::is_sigv4){
string date = get_date_rfc850();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL); requestHeaders = curl_slist_sort_insert(requestHeaders, "Accept", NULL);
string strMD5; op = "PUT";
if(S3fsCurl::is_content_md5){ type = REQTYPE_UPLOADMULTIPOST;
strMD5 = md5base64; insertAuthHeaders();
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-MD5", strMD5.c_str());
}
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("PUT", strMD5, "", date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
string payload_hash = s3fs_sha256sum(partdata.fd, partdata.startpos, partdata.size);
insertV4Headers("PUT", path, request_uri, payload_hash);
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -3438,8 +3334,6 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_UPLOADMULTIPOST;
return 0; return 0;
} }
@ -3479,13 +3373,12 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
if(!CreateCurlHandle(true)){ if(!CreateCurlHandle(true)){
return -1; return -1;
} }
string request_uri = "partNumber=" + str(part_num) + "&uploadId=" + upload_id; query_string = "partNumber=" + str(part_num) + "&uploadId=" + upload_id;
string urlargs = "?" + request_uri; string urlargs = "?" + query_string;
string resource; string resource;
string turl; string turl;
MakeUrlResource(get_realpath(to).c_str(), resource, turl); MakeUrlResource(get_realpath(to).c_str(), resource, turl);
resource += urlargs;
turl += urlargs; turl += urlargs;
url = prepare_url(turl.c_str()); url = prepare_url(turl.c_str());
path = get_realpath(to); path = get_realpath(to);
@ -3495,12 +3388,10 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
headdata = new BodyData(); headdata = new BodyData();
// Make request headers // Make request headers
string ContentType;
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){ for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string key = lower(iter->first); string key = lower(iter->first);
string value = iter->second; string value = iter->second;
if(key == "content-type"){ if(key == "content-type"){
ContentType = value;
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
}else if(key == "x-amz-copy-source"){ }else if(key == "x-amz-copy-source"){
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str()); requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
@ -3510,18 +3401,9 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
// NOTICE: x-amz-acl, x-amz-server-side-encryption is not set! // NOTICE: x-amz-acl, x-amz-server-side-encryption is not set!
} }
if(!S3fsCurl::is_sigv4){ op = "PUT";
string date = get_date_rfc850(); type = REQTYPE_COPYMULTIPOST;
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str()); insertAuthHeaders();
if(!S3fsCurl::IsPublicBucket()){
string Signature = CalcSignatureV2("PUT", "", ContentType, date, resource);
requestHeaders = curl_slist_sort_insert(requestHeaders, "Authorization", string("AWS " + AWSAccessKeyId + ":" + Signature).c_str());
}
}else{
insertV4Headers("PUT", path, request_uri, "");
}
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -3534,8 +3416,6 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders); curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
type = REQTYPE_COPYMULTIPOST;
// request // request
S3FS_PRN_INFO3("copying... [from=%s][to=%s][part=%d]", from, to, part_num); S3FS_PRN_INFO3("copying... [from=%s][to=%s][part=%d]", from, to, part_num);
@ -4126,6 +4006,25 @@ string get_sorted_header_keys(const struct curl_slist* list)
return sorted_headers; return sorted_headers;
} }
string get_header_value(const struct curl_slist* list, const string &key)
{
if(!list){
return "";
}
for( ; list; list = list->next){
string strkey = list->data;
size_t pos;
if(string::npos != (pos = strkey.find(':', 0))){
if(0 == strcasecmp(trim(strkey.substr(0, pos)).c_str(), key.c_str())){
return trim(strkey.substr(pos+1));
}
}
}
return "";
}
string get_canonical_headers(const struct curl_slist* list) string get_canonical_headers(const struct curl_slist* list)
{ {
string canonical_headers; string canonical_headers;

7
src/curl.h
View File

@ -267,6 +267,8 @@ class S3fsCurl
int b_ssekey_pos; // backup for retrying int b_ssekey_pos; // backup for retrying
std::string b_ssevalue; // backup for retrying std::string b_ssevalue; // backup for retrying
sse_type_t b_ssetype; // backup for retrying sse_type_t b_ssetype; // backup for retrying
std::string op; // the HTTP verb of the request ("PUT", "GET", etc.)
std::string query_string; // request query string
public: public:
// constructor/destructor // constructor/destructor
@ -312,7 +314,9 @@ class S3fsCurl
bool ResetHandle(void); bool ResetHandle(void);
bool RemakeHandle(void); bool RemakeHandle(void);
bool ClearInternalData(void); bool ClearInternalData(void);
void insertV4Headers(const std::string &op, const std::string &path, const std::string &query_string, const std::string &payload_hash); void insertV4Headers();
void insertV2Headers();
void insertAuthHeaders();
std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource); std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource);
std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601); std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601);
bool GetUploadId(std::string& upload_id); bool GetUploadId(std::string& upload_id);
@ -481,6 +485,7 @@ struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* d
struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value); struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value);
std::string get_sorted_header_keys(const struct curl_slist* list); std::string get_sorted_header_keys(const struct curl_slist* list);
std::string get_canonical_headers(const struct curl_slist* list, bool only_amz = false); std::string get_canonical_headers(const struct curl_slist* list, bool only_amz = false);
std::string get_header_value(const struct curl_slist* list, const std::string &key);
bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url); bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url);
std::string prepare_url(const char* url); std::string prepare_url(const char* url);
bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& ssevalue); // implement in s3fs.cpp bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& ssevalue); // implement in s3fs.cpp