mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-06-01 08:30:51 +00:00
Fixed renaming bug with SSE
This commit is contained in:
parent
222110e153
commit
e4f85c1e08
96
src/curl.cpp
96
src/curl.cpp
|
@ -3049,14 +3049,14 @@ bool S3fsCurl::GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_t
|
||||||
return (0 == result);
|
return (0 == result);
|
||||||
}
|
}
|
||||||
|
|
||||||
bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, const std::string& input, bool is_only_c, bool is_copy)
|
bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, const std::string& input, bool is_copy)
|
||||||
{
|
{
|
||||||
std::string ssevalue = input;
|
std::string ssevalue = input;
|
||||||
switch(ssetype){
|
switch(ssetype){
|
||||||
case sse_type_t::SSE_DISABLE:
|
case sse_type_t::SSE_DISABLE:
|
||||||
return true;
|
return true;
|
||||||
case sse_type_t::SSE_S3:
|
case sse_type_t::SSE_S3:
|
||||||
if(!is_only_c){
|
if(!is_copy){
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
|
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
|
||||||
}
|
}
|
||||||
return true;
|
return true;
|
||||||
|
@ -3079,7 +3079,7 @@ bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, const std::string& input, b
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
case sse_type_t::SSE_KMS:
|
case sse_type_t::SSE_KMS:
|
||||||
if(!is_only_c){
|
if(!is_copy){
|
||||||
if(ssevalue.empty()){
|
if(ssevalue.empty()){
|
||||||
ssevalue = S3fsCurl::GetSseKmsId();
|
ssevalue = S3fsCurl::GetSseKmsId();
|
||||||
}
|
}
|
||||||
|
@ -3119,10 +3119,10 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
|
||||||
requestHeaders = NULL;
|
requestHeaders = NULL;
|
||||||
responseHeaders.clear();
|
responseHeaders.clear();
|
||||||
|
|
||||||
// requestHeaders
|
// requestHeaders(SSE-C)
|
||||||
if(0 == ssekey_pos){
|
if(0 <= static_cast<ssize_t>(ssekey_pos) && ssekey_pos < S3fsCurl::sseckeys.size()){
|
||||||
std::string md5;
|
std::string md5;
|
||||||
if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseRequestHead(sse_type_t::SSE_C, md5, true, false)){
|
if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseRequestHead(sse_type_t::SSE_C, md5, false)){
|
||||||
S3FS_PRN_ERR("Failed to set SSE-C headers for sse-c key pos(%zu)(=md5(%s)).", ssekey_pos, md5.c_str());
|
S3FS_PRN_ERR("Failed to set SSE-C headers for sse-c key pos(%zu)(=md5(%s)).", ssekey_pos, md5.c_str());
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
@ -3210,7 +3210,6 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
|
||||||
bodydata.Clear();
|
bodydata.Clear();
|
||||||
|
|
||||||
std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
|
std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
|
||||||
bool need_sse_head = (S3fsCurl::GetSseType() != sse_type_t::SSE_DISABLE);
|
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
|
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type", contype.c_str());
|
||||||
|
|
||||||
// Make request headers
|
// Make request headers
|
||||||
|
@ -3224,29 +3223,13 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
|
||||||
}else if(key == "x-amz-copy-source"){
|
}else if(key == "x-amz-copy-source"){
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
|
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
|
||||||
}else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){
|
}else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){
|
||||||
// Only copy mode.
|
// skip this header, because this header is specified after logic.
|
||||||
if(is_copy){
|
|
||||||
if(AddSseRequestHead(sse_type_t::SSE_S3, value, false, true)){
|
|
||||||
need_sse_head = false;
|
|
||||||
}else{
|
|
||||||
S3FS_PRN_WARN("Failed to insert SSE-S3 header.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){
|
}else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){
|
||||||
// Only copy mode.
|
// skip this header, because this header is specified after logic.
|
||||||
if(is_copy && !value.empty()){
|
|
||||||
if(AddSseRequestHead(sse_type_t::SSE_KMS, value, false, true)){
|
|
||||||
need_sse_head = false;
|
|
||||||
}else{
|
|
||||||
S3FS_PRN_WARN("Failed to insert SSE-KMS header.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}else if(key == "x-amz-server-side-encryption-customer-key-md5"){
|
}else if(key == "x-amz-server-side-encryption-customer-key-md5"){
|
||||||
// Only copy mode.
|
// Only copy mode.
|
||||||
if(is_copy){
|
if(is_copy){
|
||||||
if(AddSseRequestHead(sse_type_t::SSE_C, value, true, true) || AddSseRequestHead(sse_type_t::SSE_C, value, true, false)){
|
if(!AddSseRequestHead(sse_type_t::SSE_C, value, true)){
|
||||||
need_sse_head = false;
|
|
||||||
}else{
|
|
||||||
S3FS_PRN_WARN("Failed to insert SSE-C header.");
|
S3FS_PRN_WARN("Failed to insert SSE-C header.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3261,9 +3244,9 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str());
|
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str());
|
||||||
}
|
}
|
||||||
// SSE
|
// SSE
|
||||||
if(need_sse_head){
|
if(S3fsCurl::GetSseType() != sse_type_t::SSE_DISABLE){
|
||||||
std::string ssevalue;
|
std::string ssevalue;
|
||||||
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
|
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){
|
||||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3396,7 +3379,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
|
||||||
// do not add SSE for create bucket
|
// do not add SSE for create bucket
|
||||||
if(0 != strcmp(tpath, "/")){
|
if(0 != strcmp(tpath, "/")){
|
||||||
std::string ssevalue;
|
std::string ssevalue;
|
||||||
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
|
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){
|
||||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3472,9 +3455,11 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, off_t
|
||||||
range += str(start + size - 1);
|
range += str(start + size - 1);
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "Range", range.c_str());
|
requestHeaders = curl_slist_sort_insert(requestHeaders, "Range", range.c_str());
|
||||||
}
|
}
|
||||||
// SSE
|
// SSE-C
|
||||||
if(!AddSseRequestHead(ssetype, ssevalue, true, false)){
|
if(sse_type_t::SSE_C == ssetype){
|
||||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
if(!AddSseRequestHead(ssetype, ssevalue, false)){
|
||||||
|
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
op = "GET";
|
op = "GET";
|
||||||
|
@ -3661,7 +3646,6 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s
|
||||||
responseHeaders.clear();
|
responseHeaders.clear();
|
||||||
|
|
||||||
std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
|
std::string contype = S3fsCurl::LookupMimeType(std::string(tpath));
|
||||||
bool need_sse_head = (S3fsCurl::GetSseType() != sse_type_t::SSE_DISABLE);
|
|
||||||
|
|
||||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||||
std::string key = lower(iter->first);
|
std::string key = lower(iter->first);
|
||||||
|
@ -3671,29 +3655,13 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s
|
||||||
}else if(is_prefix(key.c_str(), "x-amz-meta")){
|
}else if(is_prefix(key.c_str(), "x-amz-meta")){
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
|
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
|
||||||
}else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){
|
}else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){
|
||||||
// Only copy mode.
|
// skip this header, because this header is specified after logic.
|
||||||
if(is_copy){
|
|
||||||
if(AddSseRequestHead(sse_type_t::SSE_S3, value, false, true)){
|
|
||||||
need_sse_head = false;
|
|
||||||
}else{
|
|
||||||
S3FS_PRN_WARN("Failed to insert SSE-S3 header.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){
|
}else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){
|
||||||
// Only copy mode.
|
// skip this header, because this header is specified after logic.
|
||||||
if(is_copy && !value.empty()){
|
|
||||||
if(AddSseRequestHead(sse_type_t::SSE_KMS, value, false, true)){
|
|
||||||
need_sse_head = false;
|
|
||||||
}else{
|
|
||||||
S3FS_PRN_WARN("Failed to insert SSE-KMS header.");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}else if(key == "x-amz-server-side-encryption-customer-key-md5"){
|
}else if(key == "x-amz-server-side-encryption-customer-key-md5"){
|
||||||
// Only copy mode.
|
// Only copy mode.
|
||||||
if(is_copy){
|
if(is_copy){
|
||||||
if(AddSseRequestHead(sse_type_t::SSE_C, value, true, true) || AddSseRequestHead(sse_type_t::SSE_C, value, true, false)){
|
if(!AddSseRequestHead(sse_type_t::SSE_C, value, true)){
|
||||||
need_sse_head = false;
|
|
||||||
}else{
|
|
||||||
S3FS_PRN_WARN("Failed to insert SSE-C header.");
|
S3FS_PRN_WARN("Failed to insert SSE-C header.");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3707,9 +3675,9 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, std::s
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str());
|
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", GetStorageClass().c_str());
|
||||||
}
|
}
|
||||||
// SSE
|
// SSE
|
||||||
if(need_sse_head){
|
if(S3fsCurl::GetSseType() != sse_type_t::SSE_DISABLE){
|
||||||
std::string ssevalue;
|
std::string ssevalue;
|
||||||
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
|
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){
|
||||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -3994,10 +3962,10 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
|
||||||
headdata.Clear();
|
headdata.Clear();
|
||||||
responseHeaders.clear();
|
responseHeaders.clear();
|
||||||
|
|
||||||
// SSE
|
// SSE-C
|
||||||
if(sse_type_t::SSE_C == S3fsCurl::GetSseType()){
|
if(sse_type_t::SSE_C == S3fsCurl::GetSseType()){
|
||||||
std::string ssevalue;
|
std::string ssevalue;
|
||||||
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
|
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){
|
||||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -4074,8 +4042,22 @@ int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
|
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
|
||||||
}else if(key == "x-amz-copy-source-range"){
|
}else if(key == "x-amz-copy-source-range"){
|
||||||
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
|
requestHeaders = curl_slist_sort_insert(requestHeaders, iter->first.c_str(), value.c_str());
|
||||||
|
}else if(key == "x-amz-server-side-encryption" && value != "aws:kms"){
|
||||||
|
// skip this header
|
||||||
|
}else if(key == "x-amz-server-side-encryption-aws-kms-key-id"){
|
||||||
|
// skip this header
|
||||||
|
}else if(key == "x-amz-server-side-encryption-customer-key-md5"){
|
||||||
|
if(!AddSseRequestHead(sse_type_t::SSE_C, value, true)){
|
||||||
|
S3FS_PRN_WARN("Failed to insert SSE-C header.");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// SSE-C
|
||||||
|
if(sse_type_t::SSE_C == S3fsCurl::GetSseType()){
|
||||||
|
std::string ssevalue;
|
||||||
|
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false)){
|
||||||
|
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||||
}
|
}
|
||||||
// NOTICE: x-amz-acl, x-amz-server-side-encryption is not set!
|
|
||||||
}
|
}
|
||||||
|
|
||||||
op = "PUT";
|
op = "PUT";
|
||||||
|
|
|
@ -250,6 +250,7 @@ class S3fsCurl
|
||||||
void insertV2Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token);
|
void insertV2Headers(const std::string& access_key_id, const std::string& secret_access_key, const std::string& access_token);
|
||||||
void insertIBMIAMHeaders(const std::string& access_key_id, const std::string& access_token);
|
void insertIBMIAMHeaders(const std::string& access_key_id, const std::string& access_token);
|
||||||
void insertAuthHeaders();
|
void insertAuthHeaders();
|
||||||
|
bool AddSseRequestHead(sse_type_t ssetype, const std::string& ssevalue, bool is_copy);
|
||||||
std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource, const std::string& secret_access_key, const std::string& access_token);
|
std::string CalcSignatureV2(const std::string& method, const std::string& strMD5, const std::string& content_type, const std::string& date, const std::string& resource, const std::string& secret_access_key, const std::string& access_token);
|
||||||
std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601, const std::string& secret_access_key, const std::string& access_token);
|
std::string CalcSignature(const std::string& method, const std::string& canonical_uri, const std::string& query_string, const std::string& strdate, const std::string& payload_hash, const std::string& date8601, const std::string& secret_access_key, const std::string& access_token);
|
||||||
int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id);
|
int UploadMultipartPostSetup(const char* tpath, int part_num, const std::string& upload_id);
|
||||||
|
@ -338,7 +339,6 @@ class S3fsCurl
|
||||||
|
|
||||||
bool GetIAMCredentials(const char* cred_url, const char* iam_v2_token, const char* ibm_secret_access_key, std::string& response);
|
bool GetIAMCredentials(const char* cred_url, const char* iam_v2_token, const char* ibm_secret_access_key, std::string& response);
|
||||||
bool GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_token, std::string& token);
|
bool GetIAMRoleFromMetaData(const char* cred_url, const char* iam_v2_token, std::string& token);
|
||||||
bool AddSseRequestHead(sse_type_t ssetype, const std::string& ssevalue, bool is_only_c, bool is_copy);
|
|
||||||
bool GetResponseCode(long& responseCode, bool from_curl_handle = true) const;
|
bool GetResponseCode(long& responseCode, bool from_curl_handle = true) const;
|
||||||
int RequestPerform(bool dontAddAuthHeaders=false);
|
int RequestPerform(bool dontAddAuthHeaders=false);
|
||||||
int DeleteRequest(const char* tpath);
|
int DeleteRequest(const char* tpath);
|
||||||
|
|
44
src/s3fs.cpp
44
src/s3fs.cpp
|
@ -1346,26 +1346,38 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
|
||||||
FdEntity* ent;
|
FdEntity* ent;
|
||||||
if(NULL == (ent = autoent.OpenExistFdEntity(from))){
|
if(NULL == (ent = autoent.OpenExistFdEntity(from))){
|
||||||
// no opened fd
|
// no opened fd
|
||||||
|
|
||||||
|
// get mtime/ctime/atime from meta
|
||||||
|
struct timespec mtime = get_mtime(meta);
|
||||||
|
struct timespec ctime = get_ctime(meta);
|
||||||
|
struct timespec atime = get_atime(meta);
|
||||||
|
if(mtime.tv_sec < 0){
|
||||||
|
mtime.tv_sec = 0L;
|
||||||
|
mtime.tv_nsec = 0L;
|
||||||
|
}
|
||||||
|
if(ctime.tv_sec < 0){
|
||||||
|
ctime.tv_sec = 0L;
|
||||||
|
ctime.tv_nsec = 0L;
|
||||||
|
}
|
||||||
|
if(atime.tv_sec < 0){
|
||||||
|
atime.tv_sec = 0L;
|
||||||
|
atime.tv_nsec = 0L;
|
||||||
|
}
|
||||||
|
|
||||||
if(FdManager::IsCacheDir()){
|
if(FdManager::IsCacheDir()){
|
||||||
// create cache file if be needed
|
// create cache file if be needed
|
||||||
ent = autoent.Open(from, &meta, buf.st_size, S3FS_OMIT_TS, O_RDONLY, false, true, false, AutoLock::NONE);
|
//
|
||||||
|
// [NOTE]
|
||||||
|
// Do not specify "S3FS_OMIT_TS" for mctime parameter.
|
||||||
|
// This is because if the cache file does not exist, the pagelist for it
|
||||||
|
// will be recreated, but the entire file area indicated by this pagelist
|
||||||
|
// will be in the "modified" state.
|
||||||
|
// This does not affect the rename process, but the cache information in
|
||||||
|
// the "modified" state remains, making it impossible to read the file correctly.
|
||||||
|
//
|
||||||
|
ent = autoent.Open(from, &meta, buf.st_size, mtime, O_RDONLY, false, true, false, AutoLock::NONE);
|
||||||
}
|
}
|
||||||
if(ent){
|
if(ent){
|
||||||
struct timespec mtime = get_mtime(meta);
|
|
||||||
struct timespec ctime = get_ctime(meta);
|
|
||||||
struct timespec atime = get_atime(meta);
|
|
||||||
if(mtime.tv_sec < 0){
|
|
||||||
mtime.tv_sec = 0L;
|
|
||||||
mtime.tv_nsec = 0L;
|
|
||||||
}
|
|
||||||
if(ctime.tv_sec < 0){
|
|
||||||
ctime.tv_sec = 0L;
|
|
||||||
ctime.tv_nsec = 0L;
|
|
||||||
}
|
|
||||||
if(atime.tv_sec < 0){
|
|
||||||
atime.tv_sec = 0L;
|
|
||||||
atime.tv_nsec = 0L;
|
|
||||||
}
|
|
||||||
ent->SetMCtime(mtime, ctime);
|
ent->SetMCtime(mtime, ctime);
|
||||||
ent->SetAtime(atime);
|
ent->SetAtime(atime);
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue
Block a user