Sign request immediately before sending

Previously s3fs could create a long list of pre-signed requests which
could take longer than the default S3 clock skew limit of 15 minutes.
This also changes SHA-256 computation from single- to multi-threaded
since this is now computed in the worker threads.  Regression from
88cd8feb05.  Fixes #1019.
This commit is contained in:
Andrew Gaul 2019-07-02 11:29:38 -07:00
parent ae4bcd405c
commit 461a346bf4
4 changed files with 18 additions and 135 deletions

View File

@ -1582,7 +1582,6 @@ bool S3fsCurl::UploadMultipartPostSetCurlOpts(S3fsCurl* s3fscurl)
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(s3fscurl->partdata.size)); // Content-Length curl_easy_setopt(s3fscurl->hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(s3fscurl->partdata.size)); // Content-Length
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_READFUNCTION, UploadReadCallback); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_READFUNCTION, UploadReadCallback);
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_READDATA, (void*)s3fscurl); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_READDATA, (void*)s3fscurl);
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HTTPHEADER, s3fscurl->requestHeaders);
S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent
return true; return true;
@ -1604,7 +1603,6 @@ bool S3fsCurl::CopyMultipartPostSetCurlOpts(S3fsCurl* s3fscurl)
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, (void*)(&s3fscurl->headdata)); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, (void*)(&s3fscurl->headdata));
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback);
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_INFILESIZE, 0); // Content-Length curl_easy_setopt(s3fscurl->hCurl, CURLOPT_INFILESIZE, 0); // Content-Length
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HTTPHEADER, s3fscurl->requestHeaders);
S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent
return true; return true;
@ -1620,7 +1618,6 @@ bool S3fsCurl::PreGetObjectRequestSetCurlOpts(S3fsCurl* s3fscurl)
} }
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str()); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str());
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HTTPHEADER, s3fscurl->requestHeaders);
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEFUNCTION, DownloadWriteCallback); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEFUNCTION, DownloadWriteCallback);
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEDATA, (void*)s3fscurl); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_WRITEDATA, (void*)s3fscurl);
S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent S3fsCurl::AddUserAgent(s3fscurl->hCurl); // put User-Agent
@ -1640,7 +1637,6 @@ bool S3fsCurl::PreHeadRequestSetCurlOpts(S3fsCurl* s3fscurl)
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str()); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_URL, s3fscurl->url.c_str());
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_NOBODY, true); // HEAD curl_easy_setopt(s3fscurl->hCurl, CURLOPT_NOBODY, true); // HEAD
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_FILETIME, true); // Last-Modified curl_easy_setopt(s3fscurl->hCurl, CURLOPT_FILETIME, true); // Last-Modified
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HTTPHEADER, s3fscurl->requestHeaders);
// responseHeaders // responseHeaders
curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, (void*)&(s3fscurl->responseHeaders)); curl_easy_setopt(s3fscurl->hCurl, CURLOPT_HEADERDATA, (void*)&(s3fscurl->responseHeaders));
@ -1947,6 +1943,13 @@ bool S3fsCurl::CreateCurlHandle(bool only_pool, bool remake)
} }
} }
ResetHandle();
return true;
}
bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data)
{
// [NOTE] // [NOTE]
// If type is REQTYPE_IAMCRED or REQTYPE_IAMROLE, do not clear type. // If type is REQTYPE_IAMCRED or REQTYPE_IAMROLE, do not clear type.
// Because that type only uses HTTP protocol, then the special // Because that type only uses HTTP protocol, then the special
@ -1956,13 +1959,6 @@ bool S3fsCurl::CreateCurlHandle(bool only_pool, bool remake)
type = REQTYPE_UNSET; type = REQTYPE_UNSET;
} }
ResetHandle();
return true;
}
bool S3fsCurl::DestroyCurlHandle(bool restore_pool, bool clear_internal_data)
{
if(clear_internal_data){ if(clear_internal_data){
ClearInternalData(); ClearInternalData();
} }
@ -2085,14 +2081,12 @@ bool S3fsCurl::RemakeHandle()
case REQTYPE_DELETE: case REQTYPE_DELETE:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_HEAD: case REQTYPE_HEAD:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_NOBODY, true); curl_easy_setopt(hCurl, CURLOPT_NOBODY, true);
curl_easy_setopt(hCurl, CURLOPT_FILETIME, true); curl_easy_setopt(hCurl, CURLOPT_FILETIME, true);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
// responseHeaders // responseHeaders
curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&responseHeaders); curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&responseHeaders);
curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback); curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, HeaderCallback);
@ -2104,7 +2098,6 @@ bool S3fsCurl::RemakeHandle()
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0); curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_PUT: case REQTYPE_PUT:
@ -2112,7 +2105,6 @@ bool S3fsCurl::RemakeHandle()
curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true); curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true);
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
if(b_infile){ if(b_infile){
curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size));
curl_easy_setopt(hCurl, CURLOPT_INFILE, b_infile); curl_easy_setopt(hCurl, CURLOPT_INFILE, b_infile);
@ -2123,7 +2115,6 @@ bool S3fsCurl::RemakeHandle()
case REQTYPE_GET: case REQTYPE_GET:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, S3fsCurl::DownloadWriteCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, S3fsCurl::DownloadWriteCallback);
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)this); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)this);
break; break;
@ -2132,14 +2123,12 @@ bool S3fsCurl::RemakeHandle()
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_LISTBUCKET: case REQTYPE_LISTBUCKET:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_PREMULTIPOST: case REQTYPE_PREMULTIPOST:
@ -2148,12 +2137,10 @@ bool S3fsCurl::RemakeHandle()
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, 0); curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, 0);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_COMPLETEMULTIPOST: case REQTYPE_COMPLETEMULTIPOST:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
curl_easy_setopt(hCurl, CURLOPT_POST, true); curl_easy_setopt(hCurl, CURLOPT_POST, true);
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
@ -2172,7 +2159,6 @@ bool S3fsCurl::RemakeHandle()
curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(partdata.size)); curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(partdata.size));
curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::UploadReadCallback); curl_easy_setopt(hCurl, CURLOPT_READFUNCTION, S3fsCurl::UploadReadCallback);
curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this); curl_easy_setopt(hCurl, CURLOPT_READDATA, (void*)this);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_COPYMULTIPOST: case REQTYPE_COPYMULTIPOST:
@ -2183,21 +2169,18 @@ bool S3fsCurl::RemakeHandle()
curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&headdata); curl_easy_setopt(hCurl, CURLOPT_HEADERDATA, (void*)&headdata);
curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_HEADERFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0); curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_MULTILIST: case REQTYPE_MULTILIST:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_IAMCRED: case REQTYPE_IAMCRED:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
if(S3fsCurl::is_ibm_iam_auth){ if(S3fsCurl::is_ibm_iam_auth){
curl_easy_setopt(hCurl, CURLOPT_POST, true); curl_easy_setopt(hCurl, CURLOPT_POST, true);
curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast<curl_off_t>(postdata_remaining)); curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, static_cast<curl_off_t>(postdata_remaining));
@ -2209,7 +2192,6 @@ bool S3fsCurl::RemakeHandle()
case REQTYPE_ABORTMULTIUPLOAD: case REQTYPE_ABORTMULTIUPLOAD:
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
break; break;
case REQTYPE_IAMROLE: case REQTYPE_IAMROLE:
@ -2242,6 +2224,9 @@ int S3fsCurl::RequestPerform()
long responseCode; long responseCode;
int result = S3FSCURL_PERFORM_RESULT_NOTSET; int result = S3FSCURL_PERFORM_RESULT_NOTSET;
insertAuthHeaders();
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
// 1 attempt + retries... // 1 attempt + retries...
for(int retrycnt = 0; S3FSCURL_PERFORM_RESULT_NOTSET == result && retrycnt < S3fsCurl::retries; ++retrycnt){ for(int retrycnt = 0; S3FSCURL_PERFORM_RESULT_NOTSET == result && retrycnt < S3fsCurl::retries; ++retrycnt){
// Reset response code // Reset response code
@ -2688,11 +2673,9 @@ int S3fsCurl::DeleteRequest(const char* tpath)
op = "DELETE"; op = "DELETE";
type = REQTYPE_DELETE; type = REQTYPE_DELETE;
insertAuthHeaders();
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
return RequestPerform(); return RequestPerform();
@ -2758,7 +2741,6 @@ int S3fsCurl::GetIAMCredentials()
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
int result = RequestPerform(); int result = RequestPerform();
@ -2882,7 +2864,6 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
op = "HEAD"; op = "HEAD";
type = REQTYPE_HEAD; type = REQTYPE_HEAD;
insertAuthHeaders();
// set lazy function // set lazy function
fpLazySetup = PreHeadRequestSetCurlOpts; fpLazySetup = PreHeadRequestSetCurlOpts;
@ -3019,7 +3000,6 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
op = "PUT"; op = "PUT";
type = REQTYPE_PUTHEAD; type = REQTYPE_PUTHEAD;
insertAuthHeaders();
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -3027,7 +3007,6 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0); // Content-Length curl_easy_setopt(hCurl, CURLOPT_INFILESIZE, 0); // Content-Length
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
S3FS_PRN_INFO3("copying... [path=%s]", tpath); S3FS_PRN_INFO3("copying... [path=%s]", tpath);
@ -3148,14 +3127,12 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
op = "PUT"; op = "PUT";
type = REQTYPE_PUT; type = REQTYPE_PUT;
insertAuthHeaders();
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true); // HTTP PUT curl_easy_setopt(hCurl, CURLOPT_UPLOAD, true); // HTTP PUT
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
if(file){ if(file){
curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length curl_easy_setopt(hCurl, CURLOPT_INFILESIZE_LARGE, static_cast<curl_off_t>(st.st_size)); // Content-Length
curl_easy_setopt(hCurl, CURLOPT_INFILE, file); curl_easy_setopt(hCurl, CURLOPT_INFILE, file);
@ -3206,7 +3183,6 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
op = "GET"; op = "GET";
type = REQTYPE_GET; type = REQTYPE_GET;
insertAuthHeaders();
// set lazy function // set lazy function
fpLazySetup = PreGetObjectRequestSetCurlOpts; fpLazySetup = PreGetObjectRequestSetCurlOpts;
@ -3276,13 +3252,11 @@ int S3fsCurl::CheckBucket()
op = "GET"; op = "GET";
type = REQTYPE_CHKBUCKET; type = REQTYPE_CHKBUCKET;
insertAuthHeaders();
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
int result = RequestPerform(); int result = RequestPerform();
@ -3319,13 +3293,11 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
op = "GET"; op = "GET";
type = REQTYPE_LISTBUCKET; type = REQTYPE_LISTBUCKET;
insertAuthHeaders();
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
return RequestPerform(); return RequestPerform();
@ -3419,7 +3391,6 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
op = "POST"; op = "POST";
type = REQTYPE_PREMULTIPOST; type = REQTYPE_PREMULTIPOST;
insertAuthHeaders();
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
@ -3427,7 +3398,6 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, 0); curl_easy_setopt(hCurl, CURLOPT_POSTFIELDSIZE, 0);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
// request // request
@ -3496,11 +3466,9 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, string& upload_id,
op = "POST"; op = "POST";
type = REQTYPE_COMPLETEMULTIPOST; type = REQTYPE_COMPLETEMULTIPOST;
insertAuthHeaders();
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
curl_easy_setopt(hCurl, CURLOPT_POST, true); // POST curl_easy_setopt(hCurl, CURLOPT_POST, true); // POST
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
@ -3540,13 +3508,11 @@ int S3fsCurl::MultipartListRequest(string& body)
op = "GET"; op = "GET";
type = REQTYPE_MULTILIST; type = REQTYPE_MULTILIST;
insertAuthHeaders();
// setopt // setopt
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata); curl_easy_setopt(hCurl, CURLOPT_WRITEDATA, (void*)&bodydata);
curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback); curl_easy_setopt(hCurl, CURLOPT_WRITEFUNCTION, WriteMemoryCallback);
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
int result; int result;
@ -3583,11 +3549,9 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, string& upload_id)
op = "DELETE"; op = "DELETE";
type = REQTYPE_ABORTMULTIUPLOAD; type = REQTYPE_ABORTMULTIUPLOAD;
insertAuthHeaders();
curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str()); curl_easy_setopt(hCurl, CURLOPT_URL, url.c_str());
curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE"); curl_easy_setopt(hCurl, CURLOPT_CUSTOMREQUEST, "DELETE");
curl_easy_setopt(hCurl, CURLOPT_HTTPHEADER, requestHeaders);
S3fsCurl::AddUserAgent(hCurl); // put User-Agent S3fsCurl::AddUserAgent(hCurl); // put User-Agent
return RequestPerform(); return RequestPerform();
@ -3658,7 +3622,6 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
op = "PUT"; op = "PUT";
type = REQTYPE_UPLOADMULTIPOST; type = REQTYPE_UPLOADMULTIPOST;
insertAuthHeaders();
// set lazy function // set lazy function
fpLazySetup = UploadMultipartPostSetCurlOpts; fpLazySetup = UploadMultipartPostSetCurlOpts;
@ -3733,7 +3696,6 @@ int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_
op = "PUT"; op = "PUT";
type = REQTYPE_COPYMULTIPOST; type = REQTYPE_COPYMULTIPOST;
insertAuthHeaders();
// set lazy function // set lazy function
fpLazySetup = CopyMultipartPostSetCurlOpts; fpLazySetup = CopyMultipartPostSetCurlOpts;

View File

@ -196,17 +196,12 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
ssize_t bytes; ssize_t bytes;
unsigned char* result; unsigned char* result;
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512); memset(buf, 0, 512);
md5_init(&ctx_md5); md5_init(&ctx_md5);
for(ssize_t total = 0; total < size; total += bytes){ for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total); bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes); bytes = pread(fd, buf, bytes, start + total);
if(0 == bytes){ if(0 == bytes){
// end of file // end of file
break; break;
@ -221,11 +216,6 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
result = new unsigned char[get_md5_digest_length()]; result = new unsigned char[get_md5_digest_length()];
md5_digest(&ctx_md5, get_md5_digest_length(), result); md5_digest(&ctx_md5, get_md5_digest_length(), result);
if(-1 == lseek(fd, start, SEEK_SET)){
delete[] result;
return NULL;
}
return result; return result;
} }
@ -247,11 +237,6 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
size = static_cast<ssize_t>(st.st_size); size = static_cast<ssize_t>(st.st_size);
} }
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512); memset(buf, 0, 512);
if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_md5, GCRY_MD_MD5, 0))){ if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_md5, GCRY_MD_MD5, 0))){
S3FS_PRN_ERR("MD5 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); S3FS_PRN_ERR("MD5 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
@ -260,7 +245,7 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
for(ssize_t total = 0; total < size; total += bytes){ for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total); bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes); bytes = pread(fd, buf, bytes, start + total);
if(0 == bytes){ if(0 == bytes){
// end of file // end of file
break; break;
@ -277,11 +262,6 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
memcpy(result, gcry_md_read(ctx_md5, 0), get_md5_digest_length()); memcpy(result, gcry_md_read(ctx_md5, 0), get_md5_digest_length());
gcry_md_close(ctx_md5); gcry_md_close(ctx_md5);
if(-1 == lseek(fd, start, SEEK_SET)){
delete[] result;
return NULL;
}
return result; return result;
} }
@ -316,17 +296,12 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
ssize_t bytes; ssize_t bytes;
unsigned char* result; unsigned char* result;
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512); memset(buf, 0, 512);
sha256_init(&ctx_sha256); sha256_init(&ctx_sha256);
for(ssize_t total = 0; total < size; total += bytes){ for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total); bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes); bytes = pread(fd, buf, bytes, start + total);
if(0 == bytes){ if(0 == bytes){
// end of file // end of file
break; break;
@ -341,11 +316,6 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
result = new unsigned char[get_sha256_digest_length()]; result = new unsigned char[get_sha256_digest_length()];
sha256_digest(&ctx_sha256, get_sha256_digest_length(), result); sha256_digest(&ctx_sha256, get_sha256_digest_length(), result);
if(-1 == lseek(fd, start, SEEK_SET)){
delete[] result;
return NULL;
}
return result; return result;
} }
@ -386,11 +356,6 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
size = static_cast<ssize_t>(st.st_size); size = static_cast<ssize_t>(st.st_size);
} }
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512); memset(buf, 0, 512);
if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){ if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){
S3FS_PRN_ERR("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); S3FS_PRN_ERR("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
@ -399,7 +364,7 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
for(ssize_t total = 0; total < size; total += bytes){ for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total); bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes); bytes = pread(fd, buf, bytes, start + total);
if(0 == bytes){ if(0 == bytes){
// end of file // end of file
break; break;
@ -416,11 +381,6 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
memcpy(result, gcry_md_read(ctx_sha256, 0), get_sha256_digest_length()); memcpy(result, gcry_md_read(ctx_sha256, 0), get_sha256_digest_length());
gcry_md_close(ctx_sha256); gcry_md_close(ctx_sha256);
if(-1 == lseek(fd, start, SEEK_SET)){
delete[] result;
return NULL;
}
return result; return result;
} }

View File

@ -167,17 +167,12 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
size = static_cast<ssize_t>(st.st_size); size = static_cast<ssize_t>(st.st_size);
} }
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512); memset(buf, 0, 512);
md5ctx = PK11_CreateDigestContext(SEC_OID_MD5); md5ctx = PK11_CreateDigestContext(SEC_OID_MD5);
for(ssize_t total = 0; total < size; total += bytes){ for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total); bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes); bytes = pread(fd, buf, bytes, start + total);
if(0 == bytes){ if(0 == bytes){
// end of file // end of file
break; break;
@ -194,11 +189,6 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
PK11_DigestFinal(md5ctx, result, &md5outlen, get_md5_digest_length()); PK11_DigestFinal(md5ctx, result, &md5outlen, get_md5_digest_length());
PK11_DestroyContext(md5ctx, PR_TRUE); PK11_DestroyContext(md5ctx, PR_TRUE);
if(-1 == lseek(fd, start, SEEK_SET)){
delete[] result;
return NULL;
}
return result; return result;
} }
@ -243,17 +233,12 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
size = static_cast<ssize_t>(st.st_size); size = static_cast<ssize_t>(st.st_size);
} }
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512); memset(buf, 0, 512);
sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256); sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256);
for(ssize_t total = 0; total < size; total += bytes){ for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total); bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes); bytes = pread(fd, buf, bytes, start + total);
if(0 == bytes){ if(0 == bytes){
// end of file // end of file
break; break;
@ -270,11 +255,6 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
PK11_DigestFinal(sha256ctx, result, &sha256outlen, get_sha256_digest_length()); PK11_DigestFinal(sha256ctx, result, &sha256outlen, get_sha256_digest_length());
PK11_DestroyContext(sha256ctx, PR_TRUE); PK11_DestroyContext(sha256ctx, PR_TRUE);
if(-1 == lseek(fd, start, SEEK_SET)){
delete[] result;
return NULL;
}
return result; return result;
} }

View File

@ -239,17 +239,12 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
size = static_cast<ssize_t>(st.st_size); size = static_cast<ssize_t>(st.st_size);
} }
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512); memset(buf, 0, 512);
MD5_Init(&md5ctx); MD5_Init(&md5ctx);
for(ssize_t total = 0; total < size; total += bytes){ for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total); bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes); bytes = pread(fd, buf, bytes, start + total);
if(0 == bytes){ if(0 == bytes){
// end of file // end of file
break; break;
@ -265,11 +260,6 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
result = new unsigned char[get_md5_digest_length()]; result = new unsigned char[get_md5_digest_length()];
MD5_Final(result, &md5ctx); MD5_Final(result, &md5ctx);
if(-1 == lseek(fd, start, SEEK_SET)){
delete[] result;
return NULL;
}
return result; return result;
} }
@ -312,18 +302,13 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
size = static_cast<ssize_t>(st.st_size); size = static_cast<ssize_t>(st.st_size);
} }
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
sha256ctx = EVP_MD_CTX_create(); sha256ctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(sha256ctx, md, NULL); EVP_DigestInit_ex(sha256ctx, md, NULL);
memset(buf, 0, 512); memset(buf, 0, 512);
for(ssize_t total = 0; total < size; total += bytes){ for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total); bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes); bytes = pread(fd, buf, bytes, start + total);
if(0 == bytes){ if(0 == bytes){
// end of file // end of file
break; break;
@ -340,10 +325,6 @@ unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
EVP_DigestFinal_ex(sha256ctx, result, NULL); EVP_DigestFinal_ex(sha256ctx, result, NULL);
EVP_MD_CTX_destroy(sha256ctx); EVP_MD_CTX_destroy(sha256ctx);
if(-1 == lseek(fd, start, SEEK_SET)){
delete[] result;
return NULL;
}
return result; return result;
} }