mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-06-07 19:40:50 +00:00
fixed issue #39
This commit is contained in:
Takeshi Nakatani
parent
cbec8da9a3
commit
20b1c207be
|
|
@ -74,9 +74,10 @@ this option can not be specified with use_sse.
|
|||
use Amazon<6F>fs Server-Site Encryption or Server-Side Encryption with Customer-Provided Encryption Keys.
|
||||
this option can not be specified with use_rrs. specifying only "use_sse" or "use_sse=1" enables Server-Side Encryption.(use_sse=1 for old version)
|
||||
specifying this option with file path which has some SSE-C secret key enables Server-Side Encryption with Customer-Provided Encryption Keys.(use_sse=file)
|
||||
the file must be 600 permission. the file can have some lines, each line is one SSE-C key. the first line in file is used as Customer-Provided Encryption Keys for uploading and chnaging headers etc.
|
||||
the file must be 600 permission. the file can have some lines, each line is one SSE-C key. the first line in file is used as Customer-Provided Encryption Keys for uploading and change headers etc.
|
||||
if there are some keys after first line, those are used downloading object which are encripted by not first key.
|
||||
so that, you can keep all SSE-C keys in file, that is SSE-C key history.
|
||||
if AWSSSECKEYS environment is set, you can set SSE-C key instead of this option.
|
||||
.TP
|
||||
\fB\-o\fR passwd_file (default="")
|
||||
specify the path to the password file, which which takes precedence over the password in $HOME/.passwd-s3fs and /etc/passwd-s3fs
|
||||
|
|
|
|||
194
src/curl.cpp
194
src/curl.cpp
|
|
@ -92,6 +92,7 @@ static bool make_md5_from_string(const char* pstr, string& md5)
|
|||
return true;
|
||||
}
|
||||
|
||||
#if 0 // noused
|
||||
static string tolower_header_name(const char* head)
|
||||
{
|
||||
string::size_type pos;
|
||||
|
|
@ -105,6 +106,7 @@ static string tolower_header_name(const char* head)
|
|||
name += value;
|
||||
return name;
|
||||
}
|
||||
#endif
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Class BodyData
|
||||
|
|
@ -740,6 +742,38 @@ bool S3fsCurl::SetUseRrs(bool flag)
|
|||
return old;
|
||||
}
|
||||
|
||||
bool S3fsCurl::PushbackSseKeys(string& onekey)
|
||||
{
|
||||
onekey = trim(onekey);
|
||||
if(0 == onekey.size()){
|
||||
return false;
|
||||
}
|
||||
if('#' == onekey[0]){
|
||||
return false;
|
||||
}
|
||||
// make base64
|
||||
char* pbase64_key;
|
||||
if(NULL == (pbase64_key = s3fs_base64((unsigned char*)onekey.c_str(), onekey.length()))){
|
||||
FPRN("Failed to convert base64 from sse-c key %s", onekey.c_str());
|
||||
return false;
|
||||
}
|
||||
string base64_key = pbase64_key;
|
||||
free(pbase64_key);
|
||||
|
||||
// make MD5
|
||||
string strMd5;
|
||||
if(!make_md5_from_string(onekey.c_str(), strMd5)){
|
||||
FPRN("Could not make MD5 from SSE-C keys(%s).", onekey.c_str());
|
||||
return false;
|
||||
}
|
||||
// mapped MD5 = SSE Key
|
||||
sseckeymap_t md5map;
|
||||
md5map.clear();
|
||||
md5map[strMd5] = base64_key;
|
||||
S3fsCurl::sseckeys.push_back(md5map);
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetSseKeys(const char* filepath)
|
||||
{
|
||||
if(!filepath){
|
||||
|
|
@ -756,33 +790,7 @@ bool S3fsCurl::SetSseKeys(const char* filepath)
|
|||
|
||||
string line;
|
||||
while(getline(ssefs, line)){
|
||||
line = trim(line);
|
||||
if(0 == line.size()){
|
||||
continue;
|
||||
}
|
||||
if('#' == line[0]){
|
||||
continue;
|
||||
}
|
||||
// make base64
|
||||
char* pbase64_key;
|
||||
if(NULL == (pbase64_key = s3fs_base64((unsigned char*)line.c_str(), line.length()))){
|
||||
FPRN("Failed to convert base64 from sse-c key %s", line.c_str());
|
||||
continue;
|
||||
}
|
||||
string base64_key = pbase64_key;
|
||||
free(pbase64_key);
|
||||
|
||||
// make MD5
|
||||
string strMd5;
|
||||
if(!make_md5_from_string(line.c_str(), strMd5)){
|
||||
FPRN("Could not make MD5 from SSE-C keys(%s).", line.c_str());
|
||||
return false;
|
||||
}
|
||||
// mapped MD5 = SSE Key
|
||||
sseckeymap_t md5map;
|
||||
md5map.clear();
|
||||
md5map[strMd5] = base64_key;
|
||||
S3fsCurl::sseckeys.push_back(md5map);
|
||||
S3fsCurl::PushbackSseKeys(line);
|
||||
}
|
||||
if(0 == S3fsCurl::sseckeys.size()){
|
||||
FPRN("There is no SSE Key in file(%s).", filepath);
|
||||
|
|
@ -790,6 +798,26 @@ bool S3fsCurl::SetSseKeys(const char* filepath)
|
|||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::LoadEnvSseKeys(void)
|
||||
{
|
||||
char* envkeys = getenv("AWSSSECKEYS");
|
||||
if(NULL == envkeys){
|
||||
return false;
|
||||
}
|
||||
S3fsCurl::sseckeys.clear();
|
||||
|
||||
istringstream fullkeys(envkeys);
|
||||
string onekey;
|
||||
while(getline(fullkeys, onekey, ':')){
|
||||
S3fsCurl::PushbackSseKeys(onekey);
|
||||
}
|
||||
if(0 == S3fsCurl::sseckeys.size()){
|
||||
FPRN("There is no SSE Key in environment(AWSSSECKEYS=%s).", envkeys);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// If md5 is empty, returns first(current) sse key.
|
||||
|
|
@ -956,7 +984,7 @@ S3fsCurl* S3fsCurl::UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl)
|
|||
return newcurl;
|
||||
}
|
||||
|
||||
int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg)
|
||||
int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd)
|
||||
{
|
||||
int result;
|
||||
string upload_id;
|
||||
|
|
@ -982,7 +1010,7 @@ int S3fsCurl::ParallelMultipartUploadRequest(const char* tpath, headers_t& meta,
|
|||
return -errno;
|
||||
}
|
||||
|
||||
if(0 != (result = s3fscurl.PreMultipartPostRequest(tpath, meta, upload_id, ow_sse_flg))){
|
||||
if(0 != (result = s3fscurl.PreMultipartPostRequest(tpath, meta, upload_id, false))){
|
||||
close(fd2);
|
||||
return result;
|
||||
}
|
||||
|
|
@ -1888,7 +1916,7 @@ int S3fsCurl::GetIAMCredentials(void)
|
|||
//
|
||||
// If md5 is empty, build by first(current) sse key
|
||||
//
|
||||
bool S3fsCurl::AddSseKeyRequestHead(string& md5, bool is_copy_source)
|
||||
bool S3fsCurl::AddSseKeyRequestHead(string& md5, bool is_copy)
|
||||
{
|
||||
if(!S3fsCurl::IsSseCustomMode()){
|
||||
// Nothing to do
|
||||
|
|
@ -1896,7 +1924,7 @@ bool S3fsCurl::AddSseKeyRequestHead(string& md5, bool is_copy_source)
|
|||
}
|
||||
string sseckey;
|
||||
if(S3fsCurl::GetSseKey(md5, sseckey)){
|
||||
if(is_copy_source){
|
||||
if(is_copy){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm:AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-copy-source-server-side-encryption-customer-key:" + sseckey).c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-copy-source-server-side-encryption-customer-key-md5:" + md5).c_str());
|
||||
|
|
@ -2033,7 +2061,10 @@ int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta)
|
|||
return 0;
|
||||
}
|
||||
|
||||
int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg)
|
||||
//TEST
|
||||
//int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg)
|
||||
int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
|
||||
//TEST
|
||||
{
|
||||
FPRNNN("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
|
|
@ -2070,15 +2101,15 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg
|
|||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
// skip this header, because this header is specified after logic.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Not need to check error.
|
||||
if(!AddSseKeyRequestHead(value, ow_sse_flg)){ // ow_sse_flg=true means copy source
|
||||
}else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Only copy mode.
|
||||
if(!AddSseKeyRequestHead(value, is_copy)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}else if(!ow_sse_flg && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}
|
||||
}
|
||||
// "x-amz-acl", rrs, sse
|
||||
|
|
@ -2086,14 +2117,12 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg
|
|||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
|
|
@ -2126,7 +2155,10 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg
|
|||
return result;
|
||||
}
|
||||
|
||||
int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg)
|
||||
//TEST
|
||||
//int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg)
|
||||
int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
|
||||
//TEST
|
||||
{
|
||||
struct stat st;
|
||||
FILE* file = NULL;
|
||||
|
|
@ -2186,15 +2218,12 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
|||
// not set value, but after set it.
|
||||
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
// skip this header, because this header is specified after logic.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
// skip this header, because this header is specified after logic.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Not need to check error.
|
||||
if(!AddSseKeyRequestHead(value, ow_sse_flg)){ // ow_sse_flg=true means copy source
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}else if(!ow_sse_flg && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
// skip this header, because this header is specified after logic.
|
||||
}
|
||||
}
|
||||
// "x-amz-acl", rrs, sse
|
||||
|
|
@ -2202,14 +2231,12 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
|||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
|
|
@ -2438,7 +2465,10 @@ int S3fsCurl::ListBucketRequest(const char* tpath, const char* query)
|
|||
// Date: Mon, 1 Nov 2010 20:34:56 GMT
|
||||
// Authorization: AWS VGhpcyBtZXNzYWdlIHNpZ25lZCBieSBlbHZpbmc=
|
||||
//
|
||||
int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string& upload_id, bool ow_sse_flg)
|
||||
//TEST
|
||||
//int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string& upload_id, bool ow_sse_flg)
|
||||
int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string& upload_id, bool is_copy)
|
||||
//TEST
|
||||
{
|
||||
FPRNNN("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
|
|
@ -2475,15 +2505,15 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
|
|||
// not set value, but after set it.
|
||||
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
// skip this header, because this header is specified after logic.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Not need to check error.
|
||||
if(!AddSseKeyRequestHead(value, ow_sse_flg)){ // ow_sse_flg=true means copy source
|
||||
}else if(is_copy && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Only copy mode.
|
||||
if(!AddSseKeyRequestHead(value, is_copy)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}else if(!ow_sse_flg && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}
|
||||
}
|
||||
// "x-amz-acl", rrs, sse
|
||||
|
|
@ -2491,14 +2521,12 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
|
|||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
|
|
@ -2896,7 +2924,10 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
|
|||
return result;
|
||||
}
|
||||
|
||||
int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta)
|
||||
//TEST
|
||||
//int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta)
|
||||
int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy)
|
||||
//TEST
|
||||
{
|
||||
int result;
|
||||
string upload_id;
|
||||
|
|
@ -2907,7 +2938,7 @@ int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& met
|
|||
|
||||
FPRNNN("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, false))){
|
||||
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, is_copy))){
|
||||
return result;
|
||||
}
|
||||
DestroyCurlHandle();
|
||||
|
|
@ -2933,7 +2964,10 @@ int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& met
|
|||
return 0;
|
||||
}
|
||||
|
||||
int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg)
|
||||
//TEST
|
||||
//int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg)
|
||||
int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool is_copy)
|
||||
//TEST
|
||||
{
|
||||
int result;
|
||||
string upload_id;
|
||||
|
|
@ -2959,7 +2993,7 @@ int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd,
|
|||
return -errno;
|
||||
}
|
||||
|
||||
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, ow_sse_flg))){
|
||||
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, is_copy))){
|
||||
close(fd2);
|
||||
return result;
|
||||
}
|
||||
|
|
@ -3012,7 +3046,7 @@ int S3fsCurl::MultipartRenameRequest(const char* from, const char* to, headers_t
|
|||
meta["Content-Type"] = S3fsCurl::LookupMimeType(string(to));
|
||||
meta["x-amz-copy-source"] = srcresource;
|
||||
|
||||
if(0 != (result = PreMultipartPostRequest(to, meta, upload_id, false))){
|
||||
if(0 != (result = PreMultipartPostRequest(to, meta, upload_id, true))){
|
||||
return result;
|
||||
}
|
||||
DestroyCurlHandle();
|
||||
|
|
|
|||
16
src/curl.h
16
src/curl.h
|
|
@ -219,6 +219,7 @@ class S3fsCurl
|
|||
|
||||
static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
|
||||
static bool SetIAMCredentials(const char* response);
|
||||
static bool PushbackSseKeys(std::string& onekey);
|
||||
|
||||
// methods
|
||||
bool ResetHandle(void);
|
||||
|
|
@ -228,7 +229,7 @@ class S3fsCurl
|
|||
bool GetUploadId(std::string& upload_id);
|
||||
int GetIAMCredentials(void);
|
||||
|
||||
int PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id, bool ow_sse_flg);
|
||||
int PreMultipartPostRequest(const char* tpath, headers_t& meta, std::string& upload_id, bool is_copy);
|
||||
int CompleteMultipartPostRequest(const char* tpath, std::string& upload_id, etaglist_t& parts);
|
||||
int UploadMultipartPostSetup(const char* tpath, int part_num, std::string& upload_id);
|
||||
int UploadMultipartPostRequest(const char* tpath, int part_num, std::string& upload_id);
|
||||
|
|
@ -238,7 +239,7 @@ class S3fsCurl
|
|||
// class methods
|
||||
static bool InitS3fsCurl(const char* MimeFile = NULL);
|
||||
static bool DestroyS3fsCurl(void);
|
||||
static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
|
||||
static int ParallelMultipartUploadRequest(const char* tpath, headers_t& meta, int fd);
|
||||
static int ParallelGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size);
|
||||
static bool CheckIAMCredentialUpdate(void);
|
||||
|
||||
|
|
@ -256,6 +257,7 @@ class S3fsCurl
|
|||
static bool SetUseRrs(bool flag);
|
||||
static bool GetUseRrs(void) { return S3fsCurl::is_use_rrs; }
|
||||
static bool SetSseKeys(const char* filepath);
|
||||
static bool LoadEnvSseKeys(void);
|
||||
static bool GetSseKey(std::string& md5, std::string& ssekey);
|
||||
static bool GetSseKeyMd5(int pos, std::string& md5);
|
||||
static int GetSseKeyCount(void);
|
||||
|
|
@ -282,7 +284,7 @@ class S3fsCurl
|
|||
bool CreateCurlHandle(bool force = false);
|
||||
bool DestroyCurlHandle(void);
|
||||
|
||||
bool AddSseKeyRequestHead(std::string& md5, bool is_copy_source);
|
||||
bool AddSseKeyRequestHead(std::string& md5, bool is_copy);
|
||||
bool GetResponseCode(long& responseCode);
|
||||
int RequestPerform(void);
|
||||
int DeleteRequest(const char* tpath);
|
||||
|
|
@ -291,16 +293,16 @@ class S3fsCurl
|
|||
return PreHeadRequest(tpath.c_str(), bpath.c_str(), savedpath.c_str(), ssekey_pos);
|
||||
}
|
||||
int HeadRequest(const char* tpath, headers_t& meta);
|
||||
int PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg);
|
||||
int PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
|
||||
int PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy);
|
||||
int PutRequest(const char* tpath, headers_t& meta, int fd);
|
||||
int PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, std::string& ssekeymd5);
|
||||
int GetObjectRequest(const char* tpath, int fd, off_t start = -1, ssize_t size = -1);
|
||||
int CheckBucket(void);
|
||||
int ListBucketRequest(const char* tpath, const char* query);
|
||||
int MultipartListRequest(std::string& body);
|
||||
int AbortMultipartUpload(const char* tpath, std::string& upload_id);
|
||||
int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta);
|
||||
int MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
|
||||
int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy);
|
||||
int MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool is_copy);
|
||||
int MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size);
|
||||
|
||||
// methods(valiables)
|
||||
|
|
|
|||
|
|
@ -855,7 +855,7 @@ bool FdEntity::LoadFull(off_t* size, bool force_load)
|
|||
return true;
|
||||
}
|
||||
|
||||
int FdEntity::RowFlush(const char* tpath, headers_t& meta, bool ow_sse_flg, bool force_sync)
|
||||
int FdEntity::RowFlush(const char* tpath, headers_t& meta, bool force_sync)
|
||||
{
|
||||
int result;
|
||||
|
||||
|
|
@ -902,13 +902,13 @@ int FdEntity::RowFlush(const char* tpath, headers_t& meta, bool ow_sse_flg, bool
|
|||
if(120 > S3fsCurl::GetReadwriteTimeout()){
|
||||
backup = S3fsCurl::SetReadwriteTimeout(120);
|
||||
}
|
||||
result = S3fsCurl::ParallelMultipartUploadRequest(tpath ? tpath : path.c_str(), meta, fd, ow_sse_flg);
|
||||
result = S3fsCurl::ParallelMultipartUploadRequest(tpath ? tpath : path.c_str(), meta, fd);
|
||||
if(0 != backup){
|
||||
S3fsCurl::SetReadwriteTimeout(backup);
|
||||
}
|
||||
}else{
|
||||
S3fsCurl s3fscurl(true);
|
||||
result = s3fscurl.PutRequest(tpath ? tpath : path.c_str(), meta, fd, ow_sse_flg);
|
||||
result = s3fscurl.PutRequest(tpath ? tpath : path.c_str(), meta, fd);
|
||||
}
|
||||
|
||||
// seek to head of file.
|
||||
|
|
|
|||
|
|
@ -110,8 +110,8 @@ class FdEntity
|
|||
bool SetAllDisable(void) { return SetAllStatus(false); }
|
||||
bool LoadFull(off_t* size = NULL, bool force_load = false);
|
||||
int Load(off_t start, off_t size);
|
||||
int RowFlush(const char* tpath, headers_t& meta, bool ow_sse_flg, bool force_sync = false);
|
||||
int Flush(headers_t& meta, bool ow_sse_flg, bool force_sync = false) { return RowFlush(NULL, meta, ow_sse_flg, force_sync); }
|
||||
int RowFlush(const char* tpath, headers_t& meta, bool force_sync = false);
|
||||
int Flush(headers_t& meta, bool force_sync = false) { return RowFlush(NULL, meta, force_sync); }
|
||||
ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false);
|
||||
ssize_t Write(const char* bytes, off_t start, size_t size);
|
||||
};
|
||||
|
|
|
|||
53
src/s3fs.cpp
53
src/s3fs.cpp
|
|
@ -135,7 +135,10 @@ static xmlChar* get_base_exp(xmlDocPtr doc, const char* exp);
|
|||
static xmlChar* get_prefix(xmlDocPtr doc);
|
||||
static xmlChar* get_next_marker(xmlDocPtr doc);
|
||||
static char* get_object_name(xmlDocPtr doc, xmlNodePtr node, const char* path);
|
||||
static int put_headers(const char* path, headers_t& meta, bool ow_sse_flg);
|
||||
//TEST
|
||||
//static int put_headers(const char* path, headers_t& meta, bool ow_sse_flg);
|
||||
static int put_headers(const char* path, headers_t& meta, bool is_copy);
|
||||
//TEST
|
||||
static int rename_large_object(const char* from, const char* to);
|
||||
static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gid);
|
||||
static int create_directory_object(const char* path, mode_t mode, time_t time, uid_t uid, gid_t gid);
|
||||
|
|
@ -668,7 +671,10 @@ static FdEntity* get_local_fent(const char* path, bool is_load)
|
|||
* ow_sse_flg is for over writing sse header by use_sse option.
|
||||
* @return fuse return code
|
||||
*/
|
||||
static int put_headers(const char* path, headers_t& meta, bool ow_sse_flg)
|
||||
//TEST
|
||||
//static int put_headers(const char* path, headers_t& meta, bool ow_sse_flg)
|
||||
static int put_headers(const char* path, headers_t& meta, bool is_copy)
|
||||
//TEST
|
||||
{
|
||||
int result;
|
||||
S3fsCurl s3fscurl(true);
|
||||
|
|
@ -683,11 +689,11 @@ static int put_headers(const char* path, headers_t& meta, bool ow_sse_flg)
|
|||
|
||||
if(buf.st_size >= FIVE_GB){
|
||||
// multipart
|
||||
if(0 != (result = s3fscurl.MultipartHeadRequest(path, buf.st_size, meta))){
|
||||
if(0 != (result = s3fscurl.MultipartHeadRequest(path, buf.st_size, meta, is_copy))){
|
||||
return result;
|
||||
}
|
||||
}else{
|
||||
if(0 != (result = s3fscurl.PutHeadRequest(path, meta, ow_sse_flg))){
|
||||
if(0 != (result = s3fscurl.PutHeadRequest(path, meta, is_copy))){
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -791,8 +797,7 @@ static int create_file_object(const char* path, mode_t mode, uid_t uid, gid_t gi
|
|||
meta["x-amz-meta-mtime"] = str(time(NULL));
|
||||
|
||||
S3fsCurl s3fscurl(true);
|
||||
return s3fscurl.PutRequest(path, meta, -1, true); // fd=-1 means for creating zero byte object.
|
||||
// overwrite sse headers, so create new file.
|
||||
return s3fscurl.PutRequest(path, meta, -1); // fd=-1 means for creating zero byte object.
|
||||
}
|
||||
|
||||
static int s3fs_mknod(const char *path, mode_t mode, dev_t rdev)
|
||||
|
|
@ -877,8 +882,7 @@ static int create_directory_object(const char* path, mode_t mode, time_t time, u
|
|||
meta["x-amz-meta-mtime"] = str(time);
|
||||
|
||||
S3fsCurl s3fscurl;
|
||||
return s3fscurl.PutRequest(tpath.c_str(), meta, -1, true); // fd=-1 means for creating zero byte object.
|
||||
// overwrite sse headers, so create new file.
|
||||
return s3fscurl.PutRequest(tpath.c_str(), meta, -1); // fd=-1 means for creating zero byte object.
|
||||
}
|
||||
|
||||
static int s3fs_mkdir(const char* path, mode_t mode)
|
||||
|
|
@ -1040,7 +1044,7 @@ static int s3fs_symlink(const char* from, const char* to)
|
|||
return -errno;
|
||||
}
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(headers, true, true))){
|
||||
if(0 != (result = ent->Flush(headers, true))){
|
||||
DPRN("could not upload tmpfile(result=%d)", result);
|
||||
}
|
||||
FdManager::get()->Close(ent);
|
||||
|
|
@ -1076,7 +1080,7 @@ static int rename_object(const char* from, const char* to)
|
|||
meta["Content-Type"] = S3fsCurl::LookupMimeType(string(to));
|
||||
meta["x-amz-metadata-directive"] = "REPLACE";
|
||||
|
||||
if(0 != (result = put_headers(to, meta, false))){
|
||||
if(0 != (result = put_headers(to, meta, true))){
|
||||
return result;
|
||||
}
|
||||
result = s3fs_unlink(from);
|
||||
|
|
@ -1117,7 +1121,7 @@ static int rename_object_nocopy(const char* from, const char* to)
|
|||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->RowFlush(to, meta, false, true))){
|
||||
if(0 != (result = ent->RowFlush(to, meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", to, result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
|
|
@ -1417,7 +1421,7 @@ static int s3fs_chmod(const char* path, mode_t mode)
|
|||
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
|
||||
meta["x-amz-metadata-directive"] = "REPLACE";
|
||||
|
||||
if(put_headers(strpath.c_str(), meta, false) != 0){
|
||||
if(put_headers(strpath.c_str(), meta, true) != 0){
|
||||
return -EIO;
|
||||
}
|
||||
StatCache::getStatCacheData()->DelStat(nowcache);
|
||||
|
|
@ -1493,7 +1497,7 @@ static int s3fs_chmod_nocopy(const char* path, mode_t mode)
|
|||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(meta, false, true))){
|
||||
if(0 != (result = ent->Flush(meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", strpath.c_str(), result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
|
|
@ -1579,7 +1583,7 @@ static int s3fs_chown(const char* path, uid_t uid, gid_t gid)
|
|||
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
|
||||
meta["x-amz-metadata-directive"] = "REPLACE";
|
||||
|
||||
if(put_headers(strpath.c_str(), meta, false) != 0){
|
||||
if(put_headers(strpath.c_str(), meta, true) != 0){
|
||||
return -EIO;
|
||||
}
|
||||
StatCache::getStatCacheData()->DelStat(nowcache);
|
||||
|
|
@ -1665,7 +1669,7 @@ static int s3fs_chown_nocopy(const char* path, uid_t uid, gid_t gid)
|
|||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(meta, false, true))){
|
||||
if(0 != (result = ent->Flush(meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", strpath.c_str(), result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
|
|
@ -1737,7 +1741,7 @@ static int s3fs_utimens(const char* path, const struct timespec ts[2])
|
|||
meta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(strpath.c_str()));
|
||||
meta["x-amz-metadata-directive"] = "REPLACE";
|
||||
|
||||
if(put_headers(strpath.c_str(), meta, false) != 0){
|
||||
if(put_headers(strpath.c_str(), meta, true) != 0){
|
||||
return -EIO;
|
||||
}
|
||||
StatCache::getStatCacheData()->DelStat(nowcache);
|
||||
|
|
@ -1822,7 +1826,7 @@ static int s3fs_utimens_nocopy(const char* path, const struct timespec ts[2])
|
|||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(meta, false, true))){
|
||||
if(0 != (result = ent->Flush(meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", strpath.c_str(), result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
|
|
@ -1873,7 +1877,7 @@ static int s3fs_truncate(const char* path, off_t size)
|
|||
}
|
||||
|
||||
// upload
|
||||
if(0 != (result = ent->Flush(meta, false, true))){
|
||||
if(0 != (result = ent->Flush(meta, true))){
|
||||
DPRN("could not upload file(%s): result=%d", path, result);
|
||||
FdManager::get()->Close(ent);
|
||||
return result;
|
||||
|
|
@ -2028,7 +2032,7 @@ static int s3fs_flush(const char* path, struct fuse_file_info* fi)
|
|||
meta["x-amz-meta-mtime"] = str(ent_mtime);
|
||||
}
|
||||
}
|
||||
result = ent->Flush(meta, true, false);
|
||||
result = ent->Flush(meta, false);
|
||||
FdManager::get()->Close(ent);
|
||||
}
|
||||
S3FS_MALLOCTRIM(0);
|
||||
|
|
@ -3571,6 +3575,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
|||
}
|
||||
const char* ssecfile = &arg[strlen("use_sse=")];
|
||||
if(0 == strcmp(ssecfile, "1")){
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
fprintf (stderr, "%s: already set SSE-C key by environment, and confrict use_sse option.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetUseSse(true);
|
||||
}else{
|
||||
// testing sse-c, try to load AES256 keys
|
||||
|
|
@ -3593,6 +3601,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
|||
fprintf(stderr, "%s: use_sse option could not be specified with use_rrs.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
fprintf (stderr, "%s: already set SSE-C key by environment, and confrict use_sse option.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetUseSse(true);
|
||||
}
|
||||
return 0;
|
||||
|
|
@ -3846,6 +3858,9 @@ int main(int argc, char* argv[])
|
|||
}
|
||||
}
|
||||
|
||||
// Load SSE-C Key from env
|
||||
S3fsCurl::LoadEnvSseKeys();
|
||||
|
||||
// clear this structure
|
||||
memset(&s3fs_oper, 0, sizeof(s3fs_oper));
|
||||
|
||||
|
|
|
|||
|
|
@ -885,12 +885,14 @@ void show_help (void)
|
|||
" Encryption Keys.(use_sse=file)\n"
|
||||
" the file must be 600 permission. the file can have some lines,\n"
|
||||
" each line is one SSE-C key. the first line in file is used as\n"
|
||||
" Customer-Provided Encryption Keys for uploading and chnaging\n"
|
||||
" Customer-Provided Encryption Keys for uploading and changing\n"
|
||||
" headers etc.\n"
|
||||
" if there are some keys after first line, those are used\n"
|
||||
" downloading object which are encripted by not first key.\n"
|
||||
" so that, you can keep all SSE-C keys in file, that is SSE-C\n"
|
||||
" key history.\n"
|
||||
" if AWSSSECKEYS environment is set, you can set SSE-C key instead\n"
|
||||
" of this option.\n"
|
||||
"\n"
|
||||
" public_bucket (default=\"\" which means disabled)\n"
|
||||
" - anonymously mount a public bucket when set to 1\n"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user