mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-11-04 20:07:54 +00:00
Merge pull request #278 from ggtakec/master
Supported for SSE KMS(#270)
This commit is contained in:
Takeshi Nakatani
commit
8f115078cd
@ -74,13 +74,29 @@ this option can not be specified with use_sse.
|
||||
this option has been replaced by new storage_class option.
|
||||
.TP
|
||||
\fB\-o\fR use_sse (default is disable)
|
||||
use Amazon's Server-Site Encryption or Server-Side Encryption with Customer-Provided Encryption Keys.
|
||||
this option can not be specified with use_rrs. specifying only "use_sse" or "use_sse=1" enables Server-Side Encryption.(use_sse=1 for old version)
|
||||
specifying this option with file path which has some SSE-C secret key enables Server-Side Encryption with Customer-Provided Encryption Keys.(use_sse=file)
|
||||
the file must be 600 permission. the file can have some lines, each line is one SSE-C key. the first line in file is used as Customer-Provided Encryption Keys for uploading and change headers etc.
|
||||
if there are some keys after first line, those are used downloading object which are encripted by not first key.
|
||||
so that, you can keep all SSE-C keys in file, that is SSE-C key history.
|
||||
if AWSSSECKEYS environment is set, you can set SSE-C key instead of this option.
|
||||
Specify three type Amazon's Server-Site Encryption: SSE-S3, SSE-C or SSE-KMS. SSE-S3 uses Amazon S3-managed encryption keys, SSE-C uses customer-provided encryption keys, and SSE-KMS uses the master key which you manage in AWS KMS.
|
||||
You can specify "use_sse" or "use_sse=1" enables SSE-S3 type (use_sse=1 is old type parameter).
|
||||
Case of setting SSE-C, you can specify "use_sse=custom", "use_sse=custom:<custom key file path>" or "use_sse=<custom key file path>"(only <custom key file path> specified is old type parameter).
|
||||
You can use "c" for short "custom".
|
||||
The custom key file must be 600 permission. The file can have some lines, each line is one SSE-C key.
|
||||
The first line in file is used as Customer-Provided Encryption Keys for uploading and changing headers etc.
|
||||
If there are some keys after first line, those are used downloading object which are encrypted by not first key.
|
||||
So that, you can keep all SSE-C keys in file, that is SSE-C key history.
|
||||
If you specify "custom"("c") without file path, you need to set custom key by load_sse_c option or AWSSSECKEYS environment.(AWSSSECKEYS environment has some SSE-C keys with ":" separator.)
|
||||
This option is used to decide the SSE type.
|
||||
So that if you do not want to encrypt a object at uploading, but you need to decrypt encrypted object at downloaing, you can use load_sse_c option instead of this option.
|
||||
For setting SSE-KMS, specify "use_sse=kmsid" or "use_sse=kmsid:<kms id>".
|
||||
You can use "k" for short "kmsid".
|
||||
If you san specify SSE-KMS type with your <kms id> in AWS KMS, you can set it after "kmsid:"(or "k:").
|
||||
If you specify only "kmsid"("k"), you need to set AWSSSEKMSID environment which value is <kms id>.
|
||||
You must be careful about that you can not use the KMS id which is not same EC2 region.
|
||||
.TP
|
||||
\fB\-o\fR load_sse_c - specify SSE-C keys
|
||||
Specify the custom-provided encription keys file path for decrypting at duwnloading.
|
||||
If you use the custom-provided encription key at uploading, you specify with "use_sse=custom".
|
||||
The file has many lines, one line means one custom key.
|
||||
So that you can keep all SSE-C keys in file, that is SSE-C key history.
|
||||
AWSSSECKEYS environment is as same as this file contents.
|
||||
.TP
|
||||
\fB\-o\fR passwd_file (default="")
|
||||
specify the path to the password file, which which takes precedence over the password in $HOME/.passwd-s3fs and /etc/passwd-s3fs
|
||||
@ -161,7 +177,7 @@ sets the url to use to access Amazon S3. If you want to use HTTPS, then you can
|
||||
.TP
|
||||
\fB\-o\fR endpoint (default="us-east-1")
|
||||
sets the endpoint to use.
|
||||
If this option is not specified, s3fs uses \"us-east-1\" region as the default.
|
||||
If this option is not specified, s3fs uses "us-east-1" region as the default.
|
||||
If the s3fs could not connect to the region specified by this option, s3fs could not run.
|
||||
But if you do not specify this option, and if you can not connect with the default region, s3fs will retry to automatically connect to the other region.
|
||||
So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server.
|
||||
|
||||
@ -75,7 +75,7 @@ enum s3fs_log_level{
|
||||
#define S3FS_LOW_LOGPRN2(level, nest, fmt, ...) \
|
||||
if(S3FS_LOG_CRIT == level || (S3FS_LOG_CRIT != debug_level && level == (debug_level & level))){ \
|
||||
if(foreground){ \
|
||||
fprintf(stdout, "%s%s" fmt "%s\n", S3FS_LOG_LEVEL_STRING(level), S3FS_LOG_NEST(nest), __VA_ARGS__); \
|
||||
fprintf(stdout, "%s%s%s(%d): " fmt "%s\n", S3FS_LOG_LEVEL_STRING(level), S3FS_LOG_NEST(nest), __func__, __LINE__, __VA_ARGS__); \
|
||||
}else{ \
|
||||
syslog(S3FS_LOG_LEVEL_TO_SYSLOG(level), "%s" fmt "%s", S3FS_LOG_NEST(nest), __VA_ARGS__); \
|
||||
} \
|
||||
|
||||
262
src/curl.cpp
262
src/curl.cpp
@ -253,7 +253,8 @@ bool S3fsCurl::is_public_bucket = false;
|
||||
string S3fsCurl::default_acl = "private";
|
||||
storage_class_t S3fsCurl::storage_class = STANDARD;
|
||||
sseckeylist_t S3fsCurl::sseckeys;
|
||||
bool S3fsCurl::is_use_sse = false;
|
||||
std::string S3fsCurl::ssekmsid = "";
|
||||
sse_type_t S3fsCurl::ssetype = SSE_DISABLE;
|
||||
bool S3fsCurl::is_content_md5 = false;
|
||||
bool S3fsCurl::is_verbose = false;
|
||||
string S3fsCurl::AWSAccessKeyId;
|
||||
@ -808,7 +809,7 @@ bool S3fsCurl::PushbackSseKeys(string& onekey)
|
||||
// make base64
|
||||
char* pbase64_key;
|
||||
if(NULL == (pbase64_key = s3fs_base64((unsigned char*)onekey.c_str(), onekey.length()))){
|
||||
S3FS_PRN_ERR("Failed to convert base64 from sse-c key %s", onekey.c_str());
|
||||
S3FS_PRN_ERR("Failed to convert base64 from SSE-C key %s", onekey.c_str());
|
||||
return false;
|
||||
}
|
||||
string base64_key = pbase64_key;
|
||||
@ -828,12 +829,29 @@ bool S3fsCurl::PushbackSseKeys(string& onekey)
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetSseKeys(const char* filepath)
|
||||
sse_type_t S3fsCurl::SetSseType(sse_type_t type)
|
||||
{
|
||||
sse_type_t old = S3fsCurl::ssetype;
|
||||
S3fsCurl::ssetype = type;
|
||||
return old;
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetSseCKeys(const char* filepath)
|
||||
{
|
||||
if(!filepath){
|
||||
S3FS_PRN_ERR("SSE-C keys filepath is empty.");
|
||||
return false;
|
||||
}
|
||||
struct stat st;
|
||||
if(0 != stat(filepath, &st)){
|
||||
S3FS_PRN_ERR("could not open use_sse keys file(%s).", filepath);
|
||||
return false;
|
||||
}
|
||||
if(st.st_mode & (S_IXUSR | S_IRWXG | S_IRWXO)){
|
||||
S3FS_PRN_ERR("use_sse keys file %s should be 0600 permissions.", filepath);
|
||||
return false;
|
||||
}
|
||||
|
||||
S3fsCurl::sseckeys.clear();
|
||||
|
||||
ifstream ssefs(filepath);
|
||||
@ -852,12 +870,54 @@ bool S3fsCurl::SetSseKeys(const char* filepath)
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetSseKmsid(const char* kmsid)
|
||||
{
|
||||
if(!kmsid || '\0' == kmsid[0]){
|
||||
S3FS_PRN_ERR("SSE-KMS kms id is empty.");
|
||||
return false;
|
||||
}
|
||||
S3fsCurl::ssekmsid = kmsid;
|
||||
return true;
|
||||
}
|
||||
|
||||
// [NOTE]
|
||||
// Because SSE is set by some options and environment,
|
||||
// this function check the integrity of the SSE data finally.
|
||||
bool S3fsCurl::FinalCheckSse(void)
|
||||
{
|
||||
if(SSE_DISABLE == S3fsCurl::ssetype){
|
||||
S3fsCurl::ssekmsid.erase();
|
||||
}else if(SSE_S3 == S3fsCurl::ssetype){
|
||||
S3fsCurl::ssekmsid.erase();
|
||||
}else if(SSE_C == S3fsCurl::ssetype){
|
||||
if(0 == S3fsCurl::sseckeys.size()){
|
||||
S3FS_PRN_ERR("sse type is SSE-C, but there is no custom key.");
|
||||
return false;
|
||||
}
|
||||
S3fsCurl::ssekmsid.erase();
|
||||
}else if(SSE_KMS == S3fsCurl::ssetype){
|
||||
if(S3fsCurl::ssekmsid.empty()){
|
||||
S3FS_PRN_ERR("sse type is SSE-KMS, but there is no specified kms id.");
|
||||
return false;
|
||||
}
|
||||
if(!S3fsCurl::IsSignatureV4()){
|
||||
S3FS_PRN_ERR("sse type is SSE-KMS, but signature type is not v4. SSE-KMS require signature v4.");
|
||||
return false;
|
||||
}
|
||||
}else{
|
||||
S3FS_PRN_ERR("sse type is unknown(%d).", S3fsCurl::ssetype);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::LoadEnvSseKeys(void)
|
||||
bool S3fsCurl::LoadEnvSseCKeys(void)
|
||||
{
|
||||
char* envkeys = getenv("AWSSSECKEYS");
|
||||
if(NULL == envkeys){
|
||||
return false;
|
||||
// nothing to do
|
||||
return true;
|
||||
}
|
||||
S3fsCurl::sseckeys.clear();
|
||||
|
||||
@ -873,6 +933,16 @@ bool S3fsCurl::LoadEnvSseKeys(void)
|
||||
return true;
|
||||
}
|
||||
|
||||
bool S3fsCurl::LoadEnvSseKmsid(void)
|
||||
{
|
||||
char* envkmsid = getenv("AWSSSEKMSID");
|
||||
if(NULL == envkmsid){
|
||||
// nothing to do
|
||||
return true;
|
||||
}
|
||||
return S3fsCurl::SetSseKmsid(envkmsid);
|
||||
}
|
||||
|
||||
//
|
||||
// If md5 is empty, returns first(current) sse key.
|
||||
//
|
||||
@ -911,18 +981,6 @@ int S3fsCurl::GetSseKeyCount(void)
|
||||
return S3fsCurl::sseckeys.size();
|
||||
}
|
||||
|
||||
bool S3fsCurl::IsSseCustomMode(void)
|
||||
{
|
||||
return (0 < S3fsCurl::sseckeys.size());
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetUseSse(bool flag)
|
||||
{
|
||||
bool old = S3fsCurl::is_use_sse;
|
||||
S3fsCurl::is_use_sse = flag;
|
||||
return old;
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetContentMd5(bool flag)
|
||||
{
|
||||
bool old = S3fsCurl::is_content_md5;
|
||||
@ -1142,8 +1200,9 @@ S3fsCurl* S3fsCurl::ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl)
|
||||
|
||||
// duplicate request(setup new curl object)
|
||||
S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
|
||||
if(0 != (result = newcurl->PreGetObjectRequest(
|
||||
s3fscurl->path.c_str(), s3fscurl->partdata.fd, s3fscurl->partdata.startpos, s3fscurl->partdata.size, s3fscurl->b_ssekey_md5))){
|
||||
if(0 != (result = newcurl->PreGetObjectRequest(s3fscurl->path.c_str(), s3fscurl->partdata.fd,
|
||||
s3fscurl->partdata.startpos, s3fscurl->partdata.size, s3fscurl->b_ssetype, s3fscurl->b_ssevalue)))
|
||||
{
|
||||
S3FS_PRN_ERR("failed downloading part setup(%d)", result);
|
||||
delete newcurl;
|
||||
return NULL;;
|
||||
@ -1157,14 +1216,13 @@ int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, s
|
||||
{
|
||||
S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd);
|
||||
|
||||
string sseckeymd5("");
|
||||
char* psseckeymd5;
|
||||
if(NULL != (psseckeymd5 = get_object_sseckey_md5(tpath))){
|
||||
sseckeymd5 = psseckeymd5;
|
||||
free(psseckeymd5);
|
||||
sse_type_t ssetype;
|
||||
string ssevalue;
|
||||
if(!get_object_sse_type(tpath, ssetype, ssevalue)){
|
||||
S3FS_PRN_WARN("Failed to get SSE type for file(%s).", SAFESTRPTR(tpath));
|
||||
}
|
||||
int result = 0;
|
||||
ssize_t remaining_bytes;
|
||||
int result = 0;
|
||||
ssize_t remaining_bytes;
|
||||
|
||||
// cycle through open fd, pulling off 10MB chunks at a time
|
||||
for(remaining_bytes = size; 0 < remaining_bytes; ){
|
||||
@ -1183,7 +1241,7 @@ int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, s
|
||||
|
||||
// s3fscurl sub object
|
||||
S3fsCurl* s3fscurl_para = new S3fsCurl();
|
||||
if(0 != (result = s3fscurl_para->PreGetObjectRequest(tpath, fd, (start + size - remaining_bytes), chunk, sseckeymd5))){
|
||||
if(0 != (result = s3fscurl_para->PreGetObjectRequest(tpath, fd, (start + size - remaining_bytes), chunk, ssetype, ssevalue))){
|
||||
S3FS_PRN_ERR("failed downloading part setup(%d)", result);
|
||||
delete s3fscurl_para;
|
||||
return result;
|
||||
@ -1327,7 +1385,7 @@ S3fsCurl::S3fsCurl(bool ahbe) :
|
||||
hCurl(NULL), path(""), base_path(""), saved_path(""), url(""), requestHeaders(NULL),
|
||||
bodydata(NULL), headdata(NULL), LastResponseCode(-1), postdata(NULL), postdata_remaining(0), is_use_ahbe(ahbe),
|
||||
retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0),
|
||||
b_ssekey_pos(-1), b_ssekey_md5("")
|
||||
b_ssekey_pos(-1), b_ssevalue(""), b_ssetype(SSE_DISABLE)
|
||||
{
|
||||
type = REQTYPE_UNSET;
|
||||
}
|
||||
@ -2110,25 +2168,35 @@ int S3fsCurl::GetIAMCredentials(void)
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// If md5 is empty, build by first(current) sse key
|
||||
//
|
||||
bool S3fsCurl::AddSseKeyRequestHead(string& md5, bool is_copy)
|
||||
bool S3fsCurl::AddSseRequestHead(sse_type_t ssetype, string& ssevalue, bool is_only_c, bool is_copy)
|
||||
{
|
||||
if(!S3fsCurl::IsSseCustomMode()){
|
||||
// Nothing to do
|
||||
return true;
|
||||
}
|
||||
string sseckey;
|
||||
if(S3fsCurl::GetSseKey(md5, sseckey)){
|
||||
if(is_copy){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm", "AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key", sseckey.c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key-md5", md5.c_str());
|
||||
if(SSE_S3 == ssetype){
|
||||
if(!is_only_c){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
|
||||
}
|
||||
}else if(SSE_C == ssetype){
|
||||
string sseckey;
|
||||
if(S3fsCurl::GetSseKey(ssevalue, sseckey)){
|
||||
if(is_copy){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm", "AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key", sseckey.c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-key-md5", ssevalue.c_str());
|
||||
}else{
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-algorithm", "AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key", sseckey.c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key-md5", ssevalue.c_str());
|
||||
}
|
||||
}else{
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-algorithm", "AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key", sseckey.c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-key-md5", md5.c_str());
|
||||
S3FS_PRN_WARN("Failed to insert SSE-C header.");
|
||||
}
|
||||
|
||||
}else if(SSE_KMS == ssetype){
|
||||
if(!is_only_c){
|
||||
if(ssevalue.empty()){
|
||||
ssevalue = S3fsCurl::GetSseKmsId();
|
||||
}
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "aws:kms");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-aws-kms-key-id", ssevalue.c_str());
|
||||
}
|
||||
}
|
||||
return true;
|
||||
@ -2138,12 +2206,12 @@ bool S3fsCurl::AddSseKeyRequestHead(string& md5, bool is_copy)
|
||||
// tpath : target path for head request
|
||||
// bpath : saved into base_path
|
||||
// savedpath : saved into saved_path
|
||||
// ssekey_pos : -1 means "not use sse", 0 - X means "use sseckey" and "sseckey position".
|
||||
// sseckey position 0 is latest key.
|
||||
// ssekey_pos : -1 means "not" SSE-C type
|
||||
// 0 - X means SSE-C type and position for SSE-C key(0 is latest key)
|
||||
//
|
||||
bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* savedpath, int ssekey_pos)
|
||||
{
|
||||
S3FS_PRN_INFO3("[tpath=%s][bpath=%s][save=%s]", SAFESTRPTR(tpath), SAFESTRPTR(bpath), SAFESTRPTR(savedpath));
|
||||
S3FS_PRN_INFO3("[tpath=%s][bpath=%s][save=%s][sseckeypos=%d]", SAFESTRPTR(tpath), SAFESTRPTR(bpath), SAFESTRPTR(savedpath), ssekey_pos);
|
||||
|
||||
if(!tpath){
|
||||
return false;
|
||||
@ -2164,10 +2232,11 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
|
||||
responseHeaders.clear();
|
||||
|
||||
// requestHeaders
|
||||
if(0 <= ssekey_pos && S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseKeyRequestHead(md5, false)){
|
||||
S3FS_PRN_WARN("Failed to set SSE-C headers for md5(%s).", md5.c_str());
|
||||
if(0 <= ssekey_pos){
|
||||
string md5("");
|
||||
if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseRequestHead(SSE_C, md5, true, false)){
|
||||
S3FS_PRN_ERR("Failed to set SSE-C headers for sse-c key pos(%d)(=md5(%s)).", ssekey_pos, md5.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
b_ssekey_pos = ssekey_pos;
|
||||
@ -2206,11 +2275,11 @@ int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta)
|
||||
|
||||
S3FS_PRN_INFO3("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
// SSE-C mode, check all sse-c key at first
|
||||
int pos;
|
||||
for(pos = 0; static_cast<size_t>(pos) < S3fsCurl::sseckeys.size(); pos++){
|
||||
if(0 != pos && !DestroyCurlHandle()){
|
||||
// At first, try to get without SSE-C headers
|
||||
if(!PreHeadRequest(tpath) || 0 != (result = RequestPerform())){
|
||||
// If has SSE-C keys, try to get with all SSE-C keys.
|
||||
for(int pos = 0; static_cast<size_t>(pos) < S3fsCurl::sseckeys.size(); pos++){
|
||||
if(!DestroyCurlHandle()){
|
||||
return result;
|
||||
}
|
||||
if(!PreHeadRequest(tpath, NULL, NULL, pos)){
|
||||
@ -2220,16 +2289,8 @@ int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta)
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(S3fsCurl::sseckeys.size() <= static_cast<size_t>(pos)){
|
||||
// If sse-c mode is enable, s3fs fails to get head request for normal and sse object.
|
||||
// So try to get head without sse-c header.
|
||||
if(!DestroyCurlHandle() || !PreHeadRequest(tpath, NULL, NULL, -1) || 0 != (result = RequestPerform())){
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
// Not sse-c mode
|
||||
if(!PreHeadRequest(tpath) || 0 != (result = RequestPerform())){
|
||||
if(0 != result){
|
||||
DestroyCurlHandle(); // not check result.
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@ -2295,8 +2356,8 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(is_copy && key == "x-amz-server-side-encryption-customer-key-md5"){
|
||||
// Only copy mode.
|
||||
if(!AddSseKeyRequestHead(value, is_copy)){
|
||||
S3FS_PRN_WARN("Failed to insert sse(-c) header.");
|
||||
if(!AddSseRequestHead(SSE_C, value, true, is_copy)){
|
||||
S3FS_PRN_WARN("Failed to insert SSE-C header.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2308,13 +2369,10 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
|
||||
} else if(STANDARD_IA == GetStorageClass()){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "STANDARD_IA");
|
||||
}
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
S3FS_PRN_WARN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
// SSE
|
||||
string ssevalue("");
|
||||
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, true, false)){
|
||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
@ -2428,13 +2486,10 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
|
||||
} else if(STANDARD_IA == GetStorageClass()){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "STANDARD_IA");
|
||||
}
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
S3FS_PRN_WARN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
// SSE
|
||||
string ssevalue("");
|
||||
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
|
||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
@ -2482,7 +2537,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
|
||||
return result;
|
||||
}
|
||||
|
||||
int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, string& ssekeymd5)
|
||||
int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, sse_type_t ssetype, string& ssevalue)
|
||||
{
|
||||
S3FS_PRN_INFO3("[tpath=%s][start=%jd][size=%zd]", SAFESTRPTR(tpath), (intmax_t)start, size);
|
||||
|
||||
@ -2509,12 +2564,10 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
|
||||
range += str(start + size - 1);
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "Range", range.c_str());
|
||||
}
|
||||
if(0 < ssekeymd5.length()){
|
||||
if(!AddSseKeyRequestHead(ssekeymd5, false)){
|
||||
S3FS_PRN_WARN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
// SSE
|
||||
if(!AddSseRequestHead(ssetype, ssevalue, true, false)){
|
||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||
}
|
||||
|
||||
if(!S3fsCurl::is_sigv4){
|
||||
string date = get_date_rfc850();
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "Date", date.c_str());
|
||||
@ -2543,7 +2596,9 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
|
||||
partdata.size = size;
|
||||
b_partdata_startpos = start;
|
||||
b_partdata_size = size;
|
||||
b_ssekey_md5 = ssekeymd5;
|
||||
b_ssetype = ssetype;
|
||||
b_ssevalue = ssevalue;
|
||||
b_ssekey_pos = -1; // not use this value for get object.
|
||||
|
||||
type = REQTYPE_GET;
|
||||
|
||||
@ -2559,13 +2614,13 @@ int S3fsCurl::GetObjectRequest(const char* tpath, int fd, off_t start, ssize_t s
|
||||
if(!tpath){
|
||||
return -1;
|
||||
}
|
||||
string sseckeymd5("");
|
||||
char* psseckeymd5;
|
||||
if(NULL != (psseckeymd5 = get_object_sseckey_md5(tpath))){
|
||||
sseckeymd5 = psseckeymd5;
|
||||
free(psseckeymd5);
|
||||
sse_type_t ssetype;
|
||||
string ssevalue;
|
||||
if(!get_object_sse_type(tpath, ssetype, ssevalue)){
|
||||
S3FS_PRN_WARN("Failed to get SSE type for file(%s).", SAFESTRPTR(tpath));
|
||||
}
|
||||
if(0 != (result = PreGetObjectRequest(tpath, fd, start, size, sseckeymd5))){
|
||||
|
||||
if(0 != (result = PreGetObjectRequest(tpath, fd, start, size, ssetype, ssevalue))){
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -2721,8 +2776,8 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(is_copy && key == "x-amz-server-side-encryption-customer-key-md5"){
|
||||
// Only copy mode.
|
||||
if(!AddSseKeyRequestHead(value, is_copy)){
|
||||
S3FS_PRN_WARN("Failed to insert sse(-c) header.");
|
||||
if(!AddSseRequestHead(SSE_C, value, false, is_copy)){
|
||||
S3FS_PRN_WARN("Failed to insert SSE-C header.");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -2733,13 +2788,10 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
|
||||
} else if(STANDARD_IA == GetStorageClass()){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "STANDARD_IA");
|
||||
}
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
S3FS_PRN_WARN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
// SSE
|
||||
string ssevalue("");
|
||||
if(!AddSseRequestHead(S3fsCurl::GetSseType(), ssevalue, false, false)){
|
||||
S3FS_PRN_WARN("Failed to set SSE header, but continue...");
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
|
||||
41
src/curl.h
41
src/curl.h
@ -122,12 +122,21 @@ typedef std::map<std::string, std::string> iamcredmap_t;
|
||||
typedef std::map<std::string, std::string> sseckeymap_t;
|
||||
typedef std::list<sseckeymap_t> sseckeylist_t;
|
||||
|
||||
// strage class(rrs)
|
||||
enum storage_class_t {
|
||||
STANDARD,
|
||||
STANDARD_IA,
|
||||
REDUCED_REDUNDANCY,
|
||||
};
|
||||
|
||||
// sse type
|
||||
enum sse_type_t {
|
||||
SSE_DISABLE = 0, // not use server side encrypting
|
||||
SSE_S3, // server side encrypting by S3 key
|
||||
SSE_C, // server side encrypting by custom key
|
||||
SSE_KMS // server side encrypting by kms id
|
||||
};
|
||||
|
||||
// share
|
||||
#define SHARE_MUTEX_DNS 0
|
||||
#define SHARE_MUTEX_SSL_SESSION 1
|
||||
@ -173,7 +182,8 @@ class S3fsCurl
|
||||
static std::string default_acl; // TODO: to enum
|
||||
static storage_class_t storage_class;
|
||||
static sseckeylist_t sseckeys;
|
||||
static bool is_use_sse;
|
||||
static std::string ssekmsid;
|
||||
static sse_type_t ssetype;
|
||||
static bool is_content_md5;
|
||||
static bool is_verbose;
|
||||
static std::string AWSAccessKeyId;
|
||||
@ -212,8 +222,9 @@ class S3fsCurl
|
||||
int b_postdata_remaining; // backup for retrying
|
||||
off_t b_partdata_startpos; // backup for retrying
|
||||
ssize_t b_partdata_size; // backup for retrying
|
||||
bool b_ssekey_pos; // backup for retrying
|
||||
std::string b_ssekey_md5; // backup for retrying
|
||||
int b_ssekey_pos; // backup for retrying
|
||||
std::string b_ssevalue; // backup for retrying
|
||||
sse_type_t b_ssetype; // backup for retrying
|
||||
|
||||
public:
|
||||
// constructor/destructor
|
||||
@ -246,6 +257,8 @@ class S3fsCurl
|
||||
|
||||
static bool ParseIAMCredentialResponse(const char* response, iamcredmap_t& keyval);
|
||||
static bool SetIAMCredentials(const char* response);
|
||||
static bool LoadEnvSseCKeys(void);
|
||||
static bool LoadEnvSseKmsid(void);
|
||||
static bool PushbackSseKeys(std::string& onekey);
|
||||
|
||||
static int CurlDebugFunc(CURL* hcurl, curl_infotype type, char* data, size_t size, void* userptr);
|
||||
@ -288,14 +301,21 @@ class S3fsCurl
|
||||
static std::string SetDefaultAcl(const char* acl);
|
||||
static storage_class_t SetStorageClass(storage_class_t storage_class);
|
||||
static storage_class_t GetStorageClass() { return S3fsCurl::storage_class; }
|
||||
static bool SetSseKeys(const char* filepath);
|
||||
static bool LoadEnvSseKeys(void);
|
||||
static bool LoadEnvSse(void) { return (S3fsCurl::LoadEnvSseCKeys() && S3fsCurl::LoadEnvSseKmsid()); }
|
||||
static sse_type_t SetSseType(sse_type_t type);
|
||||
static sse_type_t GetSseType(void) { return S3fsCurl::ssetype; }
|
||||
static bool IsSseDisable(void) { return (SSE_DISABLE == S3fsCurl::ssetype); }
|
||||
static bool IsSseS3Type(void) { return (SSE_S3 == S3fsCurl::ssetype); }
|
||||
static bool IsSseCType(void) { return (SSE_C == S3fsCurl::ssetype); }
|
||||
static bool IsSseKmsType(void) { return (SSE_KMS == S3fsCurl::ssetype); }
|
||||
static bool FinalCheckSse(void);
|
||||
static bool SetSseCKeys(const char* filepath);
|
||||
static bool SetSseKmsid(const char* kmsid);
|
||||
static bool IsSetSseKmsId(void) { return !S3fsCurl::ssekmsid.empty(); }
|
||||
static const char* GetSseKmsId(void) { return S3fsCurl::ssekmsid.c_str(); }
|
||||
static bool GetSseKey(std::string& md5, std::string& ssekey);
|
||||
static bool GetSseKeyMd5(int pos, std::string& md5);
|
||||
static int GetSseKeyCount(void);
|
||||
static bool IsSseCustomMode(void);
|
||||
static bool SetUseSse(bool flag);
|
||||
static bool GetUseSse(void) { return S3fsCurl::is_use_sse; }
|
||||
static bool SetContentMd5(bool flag);
|
||||
static bool SetVerbose(bool flag);
|
||||
static bool GetVerbose(void) { return S3fsCurl::is_verbose; }
|
||||
@ -318,7 +338,7 @@ class S3fsCurl
|
||||
bool CreateCurlHandle(bool force = false);
|
||||
bool DestroyCurlHandle(void);
|
||||
|
||||
bool AddSseKeyRequestHead(std::string& md5, bool is_copy);
|
||||
bool AddSseRequestHead(sse_type_t ssetype, std::string& ssevalue, bool is_only_c, bool is_copy);
|
||||
bool GetResponseCode(long& responseCode);
|
||||
int RequestPerform(void);
|
||||
int DeleteRequest(const char* tpath);
|
||||
@ -329,7 +349,7 @@ class S3fsCurl
|
||||
int HeadRequest(const char* tpath, headers_t& meta);
|
||||
int PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy);
|
||||
int PutRequest(const char* tpath, headers_t& meta, int fd);
|
||||
int PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, std::string& ssekeymd5);
|
||||
int PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, sse_type_t ssetype, std::string& ssevalue);
|
||||
int GetObjectRequest(const char* tpath, int fd, off_t start = -1, ssize_t size = -1);
|
||||
int CheckBucket(void);
|
||||
int ListBucketRequest(const char* tpath, const char* query);
|
||||
@ -441,6 +461,7 @@ std::string get_sorted_header_keys(const struct curl_slist* list);
|
||||
std::string get_canonical_headers(const struct curl_slist* list, bool only_amz = false);
|
||||
bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url);
|
||||
std::string prepare_url(const char* url);
|
||||
bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& ssevalue); // implement in s3fs.cpp
|
||||
|
||||
#endif // S3FS_CURL_H_
|
||||
|
||||
|
||||
227
src/s3fs.cpp
227
src/s3fs.cpp
@ -684,27 +684,35 @@ static int check_parent_object_access(const char* path, int mask)
|
||||
}
|
||||
|
||||
//
|
||||
// This function is global, is called fom curl class(GetObject).
|
||||
// ssevalue is MD5 for SSE-C type, or KMS id for SSE-KMS
|
||||
//
|
||||
char* get_object_sseckey_md5(const char* path)
|
||||
bool get_object_sse_type(const char* path, sse_type_t& ssetype, string& ssevalue)
|
||||
{
|
||||
if(!path){
|
||||
return NULL;
|
||||
return false;
|
||||
}
|
||||
headers_t meta;
|
||||
|
||||
headers_t meta;
|
||||
if(0 != get_object_attribute(path, NULL, &meta)){
|
||||
S3FS_PRN_ERR("Failed to get object(%s) headers", path);
|
||||
return NULL;
|
||||
return false;
|
||||
}
|
||||
|
||||
ssetype = SSE_DISABLE;
|
||||
ssevalue.erase();
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
return strdup((*iter).second.c_str());
|
||||
string key = (*iter).first;
|
||||
if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption") && 0 == strcasecmp((*iter).second.c_str(), "AES256")){
|
||||
ssetype = SSE_S3;
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-aws-kms-key-id")){
|
||||
ssetype = SSE_KMS;
|
||||
ssevalue = (*iter).second;
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
ssetype = SSE_C;
|
||||
ssevalue = (*iter).second;
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
return true;
|
||||
}
|
||||
|
||||
static FdEntity* get_local_fent(const char* path, bool is_load)
|
||||
@ -2229,23 +2237,19 @@ static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl)
|
||||
if(!s3fscurl){
|
||||
return NULL;
|
||||
}
|
||||
int ssec_key_pos = s3fscurl->GetLastPreHeadSeecKeyPos();
|
||||
int next_retry_count = s3fscurl->GetMultipartRetryCount() + 1;
|
||||
int ssec_key_pos= s3fscurl->GetLastPreHeadSeecKeyPos();
|
||||
int retry_count = s3fscurl->GetMultipartRetryCount();
|
||||
|
||||
if(s3fscurl->IsOverMultipartRetryCount()){
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
// If sse-c mode, start check not sse-c(ssec_key_pos = -1).
|
||||
// do increment ssec_key_pos for checking all sse-c key.
|
||||
next_retry_count = 0;
|
||||
ssec_key_pos++;
|
||||
if(S3fsCurl::GetSseKeyCount() <= ssec_key_pos){
|
||||
S3FS_PRN_ERR("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpacialSavedPath().c_str());
|
||||
return NULL;
|
||||
}
|
||||
}else{
|
||||
// retry next sse key.
|
||||
// if end of sse key, set retry master count is up.
|
||||
ssec_key_pos = (ssec_key_pos < 0 ? 0 : ssec_key_pos + 1);
|
||||
if(0 == S3fsCurl::GetSseKeyCount() || S3fsCurl::GetSseKeyCount() <= ssec_key_pos){
|
||||
if(s3fscurl->IsOverMultipartRetryCount()){
|
||||
S3FS_PRN_ERR("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpacialSavedPath().c_str());
|
||||
return NULL;
|
||||
}
|
||||
ssec_key_pos= -1;
|
||||
retry_count++;
|
||||
}
|
||||
|
||||
S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
|
||||
@ -2258,7 +2262,7 @@ static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl)
|
||||
delete newcurl;
|
||||
return NULL;
|
||||
}
|
||||
newcurl->SetMultipartRetryCount(next_retry_count);
|
||||
newcurl->SetMultipartRetryCount(retry_count);
|
||||
|
||||
return newcurl;
|
||||
}
|
||||
@ -2300,8 +2304,8 @@ static int readdir_multi_head(const char* path, S3ObjList& head, void* buf, fuse
|
||||
continue;
|
||||
}
|
||||
|
||||
// First check for directory, start checking "not sse-c".
|
||||
// If checking failed, retry to check with "sse-c" by retry callback func when sse-c mode.
|
||||
// First check for directory, start checking "not SSE-C".
|
||||
// If checking failed, retry to check with "SSE-C" by retry callback func when SSE-C mode.
|
||||
S3fsCurl* s3fscurl = new S3fsCurl();
|
||||
if(!s3fscurl->PreHeadRequest(disppath, (*iter), disppath)){ // target path = cache key path.(ex "dir/")
|
||||
S3FS_PRN_WARN("Could not make curl object for head request(%s).", disppath.c_str());
|
||||
@ -2319,8 +2323,16 @@ static int readdir_multi_head(const char* path, S3ObjList& head, void* buf, fuse
|
||||
|
||||
// Multi request
|
||||
if(0 != (result = curlmulti.Request())){
|
||||
S3FS_PRN_ERR("error occuered in multi request(errno=%d).", result);
|
||||
break;
|
||||
// If result is -EIO, it is somthing error occurred.
|
||||
// This case includes that the object is encrypting(SSE) and s3fs does not have keys.
|
||||
// So s3fs set result to 0 in order to continue the process.
|
||||
if(-EIO == result){
|
||||
S3FS_PRN_WARN("error occuered in multi request(errno=%d), but continue...", result);
|
||||
result = 0;
|
||||
}else{
|
||||
S3FS_PRN_ERR("error occuered in multi request(errno=%d).", result);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// populate fuse buffer
|
||||
@ -4246,10 +4258,6 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
if(0 == rrs){
|
||||
S3fsCurl::SetStorageClass(STANDARD);
|
||||
}else if(1 == rrs){
|
||||
if(S3fsCurl::GetUseSse()){
|
||||
S3FS_PRN_EXIT("use_rrs option could not be specified with use_sse.");
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetStorageClass(REDUCED_REDUNDANCY);
|
||||
}else{
|
||||
S3FS_PRN_EXIT("poorly formed argument to option: use_rrs");
|
||||
@ -4264,10 +4272,6 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
}else if(0 == strcmp(storage_class, "standard_ia")){
|
||||
S3fsCurl::SetStorageClass(STANDARD_IA);
|
||||
}else if(0 == strcmp(storage_class, "reduced_redundancy")){
|
||||
if(S3fsCurl::GetUseSse()){
|
||||
S3FS_PRN_EXIT("storage class reduced_redundancy option could not be specified with use_sse.");
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetStorageClass(REDUCED_REDUNDANCY);
|
||||
}else{
|
||||
S3FS_PRN_EXIT("unknown value for storage_class: %s", storage_class);
|
||||
@ -4275,48 +4279,114 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
if(0 == strcmp(arg, "use_sse") || 0 == STR2NCMP(arg, "use_sse=")){
|
||||
if(0 == STR2NCMP(arg, "use_sse=")){
|
||||
if(REDUCED_REDUNDANCY == S3fsCurl::GetStorageClass()){
|
||||
S3FS_PRN_EXIT("use_sse option could not be specified with storage class reduced_redundancy.");
|
||||
//
|
||||
// [NOTE]
|
||||
// use_sse Set Server Side Encrypting type to SSE-S3
|
||||
// use_sse=1
|
||||
// use_sse=file Set Server Side Encrypting type to Custom key(SSE-C) and load custom keys
|
||||
// use_sse=custom(c):file
|
||||
// use_sse=custom(c) Set Server Side Encrypting type to Custom key(SSE-C)
|
||||
// use_sse=kmsid(k):kms-key-id Set Server Side Encrypting type to AWS Key Management key id(SSE-KMS) and load KMS id
|
||||
// use_sse=kmsid(k) Set Server Side Encrypting type to AWS Key Management key id(SSE-KMS)
|
||||
//
|
||||
// load_sse_c=file Load Server Side Encrypting custom keys
|
||||
//
|
||||
// AWSSSECKEYS Loaing Environment for Server Side Encrypting custom keys
|
||||
// AWSSSEKMSID Loaing Environment for Server Side Encrypting Key id
|
||||
//
|
||||
if(0 == STR2NCMP(arg, "use_sse")){
|
||||
if(0 == strcmp(arg, "use_sse") || 0 == strcmp(arg, "use_sse=1")){ // use_sse=1 is old type paraemter
|
||||
// sse type is SSE_S3
|
||||
if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseS3Type()){
|
||||
S3FS_PRN_EXIT("already set SSE another type, so confrict use_sse option or environment.");
|
||||
return -1;
|
||||
}
|
||||
const char* ssecfile = &arg[strlen("use_sse=")];
|
||||
if(0 == strcmp(ssecfile, "1")){
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
S3FS_PRN_EXIT("already set SSE-C key by environment, and confrict use_sse option.");
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetUseSse(true);
|
||||
S3fsCurl::SetSseType(SSE_S3);
|
||||
|
||||
}else if(0 == strcmp(arg, "use_sse=kmsid") || 0 == strcmp(arg, "use_sse=k")){
|
||||
// sse type is SSE_KMS with out kmsid(expecting id is loaded by environment)
|
||||
if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseKmsType()){
|
||||
S3FS_PRN_EXIT("already set SSE another type, so confrict use_sse option or environment.");
|
||||
return -1;
|
||||
}
|
||||
if(!S3fsCurl::IsSetSseKmsId()){
|
||||
S3FS_PRN_EXIT("use_sse=kms but not loaded kms id by environemnt.");
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetSseType(SSE_KMS);
|
||||
|
||||
}else if(0 == STR2NCMP(arg, "use_sse=kmsid:") || 0 == STR2NCMP(arg, "use_sse=k:")){
|
||||
// sse type is SSE_KMS with kmsid
|
||||
if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseKmsType()){
|
||||
S3FS_PRN_EXIT("already set SSE another type, so confrict use_sse option or environment.");
|
||||
return -1;
|
||||
}
|
||||
const char* kmsid;
|
||||
if(0 == STR2NCMP(arg, "use_sse=kmsid:")){
|
||||
kmsid = &arg[strlen("use_sse=kmsid:")];
|
||||
}else{
|
||||
// testing sse-c, try to load AES256 keys
|
||||
struct stat st;
|
||||
if(0 != stat(ssecfile, &st)){
|
||||
S3FS_PRN_EXIT("could not open use_sse keys file(%s).", ssecfile);
|
||||
return -1;
|
||||
}
|
||||
if(st.st_mode & (S_IXUSR | S_IRWXG | S_IRWXO)){
|
||||
S3FS_PRN_EXIT("use_sse keys file %s should be 0600 permissions.", ssecfile);
|
||||
return -1;
|
||||
}
|
||||
if(!S3fsCurl::SetSseKeys(ssecfile)){
|
||||
S3FS_PRN_EXIT("failed to load use_sse keys file %s.", ssecfile);
|
||||
return -1;
|
||||
}
|
||||
kmsid = &arg[strlen("use_sse=k:")];
|
||||
}
|
||||
if(!S3fsCurl::SetSseKmsid(kmsid)){
|
||||
S3FS_PRN_EXIT("failed to load use_sse kms id.");
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetSseType(SSE_KMS);
|
||||
|
||||
}else if(0 == strcmp(arg, "use_sse=custom") || 0 == strcmp(arg, "use_sse=c")){
|
||||
// sse type is SSE_C with out custom keys(expecting keays are loaded by environment or load_sse_c option)
|
||||
if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseCType()){
|
||||
S3FS_PRN_EXIT("already set SSE another type, so confrict use_sse option or environment.");
|
||||
return -1;
|
||||
}
|
||||
// [NOTE]
|
||||
// do not check ckeys exists here.
|
||||
//
|
||||
S3fsCurl::SetSseType(SSE_C);
|
||||
|
||||
}else if(0 == STR2NCMP(arg, "use_sse=custom:") || 0 == STR2NCMP(arg, "use_sse=c:")){
|
||||
// sse type is SSE_C with custom keys
|
||||
if(!S3fsCurl::IsSseDisable() && !S3fsCurl::IsSseCType()){
|
||||
S3FS_PRN_EXIT("already set SSE another type, so confrict use_sse option or environment.");
|
||||
return -1;
|
||||
}
|
||||
const char* ssecfile;
|
||||
if(0 == STR2NCMP(arg, "use_sse=custom:")){
|
||||
ssecfile = &arg[strlen("use_sse=custom:")];
|
||||
}else{
|
||||
ssecfile = &arg[strlen("use_sse=c:")];
|
||||
}
|
||||
if(!S3fsCurl::SetSseCKeys(ssecfile)){
|
||||
S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile);
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetSseType(SSE_C);
|
||||
|
||||
}else if(0 == strcmp(arg, "use_sse=")){ // this type is old style(paraemter is custom key file path)
|
||||
// SSE_C with custom keys.
|
||||
const char* ssecfile = &arg[strlen("use_sse=")];
|
||||
if(!S3fsCurl::SetSseCKeys(ssecfile)){
|
||||
S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile);
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetSseType(SSE_C);
|
||||
|
||||
}else{
|
||||
if(REDUCED_REDUNDANCY == S3fsCurl::GetStorageClass()){
|
||||
S3FS_PRN_EXIT("use_sse option could not be specified with storage class reduced_redundancy.");
|
||||
return -1;
|
||||
}
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
S3FS_PRN_EXIT("already set SSE-C key by environment, and confrict use_sse option.");
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetUseSse(true);
|
||||
// never come here.
|
||||
S3FS_PRN_EXIT("something wrong use_sse optino.");
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
// [NOTE]
|
||||
// Do only load SSE custom keys, care for set without set sse type.
|
||||
if(0 == STR2NCMP(arg, "load_sse_c=")){
|
||||
const char* ssecfile = &arg[strlen("load_sse_c=")];
|
||||
if(!S3fsCurl::SetSseCKeys(ssecfile)){
|
||||
S3FS_PRN_EXIT("failed to load use_sse custom key file(%s).", ssecfile);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "ssl_verify_hostname=")){
|
||||
long sslvh = static_cast<long>(s3fs_strtoofft(strchr(arg, '=') + sizeof(char)));
|
||||
if(-1 == S3fsCurl::SetSslVerifyHostname(sslvh)){
|
||||
@ -4601,8 +4671,11 @@ int main(int argc, char* argv[])
|
||||
}
|
||||
}
|
||||
|
||||
// Load SSE-C Key from env
|
||||
S3fsCurl::LoadEnvSseKeys();
|
||||
// Load SSE environment
|
||||
if(!S3fsCurl::LoadEnvSse()){
|
||||
S3FS_PRN_EXIT("something wrong about SSE environment.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// clear this structure
|
||||
memset(&s3fs_oper, 0, sizeof(s3fs_oper));
|
||||
@ -4615,6 +4688,18 @@ int main(int argc, char* argv[])
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// [NOTE]
|
||||
// exclusive option check here.
|
||||
//
|
||||
if(REDUCED_REDUNDANCY == S3fsCurl::GetStorageClass() && !S3fsCurl::IsSseDisable()){
|
||||
S3FS_PRN_EXIT("use_sse option could not be specified with storage class reduced_redundancy.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
if(!S3fsCurl::FinalCheckSse()){
|
||||
S3FS_PRN_EXIT("something wrong about SSE options.");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
// The first plain argument is the bucket
|
||||
if(bucket.size() == 0){
|
||||
S3FS_PRN_EXIT("missing BUCKET argument.");
|
||||
|
||||
@ -84,8 +84,6 @@
|
||||
|
||||
#endif // HAVE_MALLOC_TRIM
|
||||
|
||||
char* get_object_sseckey_md5(const char* path);
|
||||
|
||||
#endif // S3FS_S3_H_
|
||||
|
||||
/*
|
||||
|
||||
@ -899,24 +899,49 @@ void show_help (void)
|
||||
" standard, standard_ia, and reduced_redundancy.\n"
|
||||
"\n"
|
||||
" use_sse (default is disable)\n"
|
||||
" - use Amazon's Server-Site Encryption or Server-Side Encryption\n"
|
||||
" with Customer-Provided Encryption Keys.\n"
|
||||
" this option can not be specified with use_rrs. specifying only \n"
|
||||
" \"use_sse\" or \"use_sse=1\" enables Server-Side Encryption.\n"
|
||||
" (use_sse=1 for old version)\n"
|
||||
" specifying this option with file path which has some SSE-C\n"
|
||||
" secret key enables Server-Side Encryption with Customer-Provided\n"
|
||||
" Encryption Keys.(use_sse=file)\n"
|
||||
" the file must be 600 permission. the file can have some lines,\n"
|
||||
" each line is one SSE-C key. the first line in file is used as\n"
|
||||
" Customer-Provided Encryption Keys for uploading and changing\n"
|
||||
" headers etc.\n"
|
||||
" if there are some keys after first line, those are used\n"
|
||||
" downloading object which are encrypted by not first key.\n"
|
||||
" so that, you can keep all SSE-C keys in file, that is SSE-C\n"
|
||||
" key history.\n"
|
||||
" if AWSSSECKEYS environment is set, you can set SSE-C key instead\n"
|
||||
" - Specify three type Amazon's Server-Site Encryption: SSE-S3,\n"
|
||||
" SSE-C or SSE-KMS. SSE-S3 uses Amazon S3-managed encryption\n"
|
||||
" keys, SSE-C uses customer-provided encryption keys, and\n"
|
||||
" SSE-KMS uses the master key which you manage in AWS KMS.\n"
|
||||
" You can specify \"use_sse\" or \"use_sse=1\" enables SSE-S3\n"
|
||||
" type(use_sse=1 is old type parameter).\n"
|
||||
" Case of setting SSE-C, you can specify \"use_sse=custom\",\n"
|
||||
" \"use_sse=custom:<custom key file path>\" or\n"
|
||||
" \"use_sse=<custom key file path>\"(only <custom key file path>\n"
|
||||
" specified is old type parameter). You can use \"c\" for\n"
|
||||
" short \"custom\".\n"
|
||||
" The custom key file must be 600 permission. The file can\n"
|
||||
" have some lines, each line is one SSE-C key. The first line\n"
|
||||
" in file is used as Customer-Provided Encryption Keys for\n"
|
||||
" uploading and changing headers etc. If there are some keys\n"
|
||||
" after first line, those are used downloading object which\n"
|
||||
" are encrypted by not first key. So that, you can keep all\n"
|
||||
" SSE-C keys in file, that is SSE-C key history.\n"
|
||||
" If you specify \"custom\"(\"c\") without file path, you\n"
|
||||
" need to set custom key by load_sse_c option or AWSSSECKEYS\n"
|
||||
" environment.(AWSSSECKEYS environment has some SSE-C keys\n"
|
||||
" with \":\" separator.) This option is used to decide the\n"
|
||||
" SSE type. So that if you do not want to encrypt a object\n"
|
||||
" object at uploading, but you need to decrypt encrypted\n"
|
||||
" object at downloaing, you can use load_sse_c option instead\n"
|
||||
" of this option.\n"
|
||||
" For setting SSE-KMS, specify \"use_sse=kmsid\" or\n"
|
||||
" \"use_sse=kmsid:<kms id>\". You can use \"k\" for short \"kmsid\".\n"
|
||||
" If you san specify SSE-KMS type with your <kms id> in AWS\n"
|
||||
" KMS, you can set it after \"kmsid:\"(or \"k:\"). If you\n"
|
||||
" specify only \"kmsid\"(\"k\"), you need to set AWSSSEKMSID\n"
|
||||
" environment which value is <kms id>. You must be careful\n"
|
||||
" about that you can not use the KMS id which is not same EC2\n"
|
||||
" region.\n"
|
||||
"\n"
|
||||
" load_sse_c - specify SSE-C keys\n"
|
||||
" Specify the custom-provided encription keys file path for decrypting\n"
|
||||
" at duwnloading.\n"
|
||||
" If you use the custom-provided encription key at uploading, you\n"
|
||||
" specify with \"use_sse=custom\". The file has many lines, one line\n"
|
||||
" means one custom key. So that you can keep all SSE-C keys in file,\n"
|
||||
" that is SSE-C key history. AWSSSECKEYS environment is as same as this\n"
|
||||
" file contents.\n"
|
||||
"\n"
|
||||
" public_bucket (default=\"\" which means disabled)\n"
|
||||
" - anonymously mount a public bucket when set to 1\n"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user