mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-11-05 04:17:52 +00:00
Support for SSE-C, issue #39
This commit is contained in:
Takeshi Nakatani
parent
95f8cab139
commit
7a55eab399
@ -71,9 +71,12 @@ this option can not be specified with use_sse.
|
||||
(can specify use_rrs=1 for old version)
|
||||
.TP
|
||||
\fB\-o\fR use_sse (default is disable)
|
||||
use Amazon's Server Site Encryption.
|
||||
this option can not be specified with use_rrs.
|
||||
(can specify use_sse=1 for old version)
|
||||
use Amazon<6F>fs Server-Site Encryption or Server-Side Encryption with Customer-Provided Encryption Keys.
|
||||
this option can not be specified with use_rrs. specifying only "use_sse" or "use_sse=1" enables Server-Side Encryption.(use_sse=1 for old version)
|
||||
specifying this option with file path which has some SSE-C secret key enables Server-Side Encryption with Customer-Provided Encryption Keys.(use_sse=file)
|
||||
the file must be 600 permission. the file can have some lines, each line is one SSE-C key. the first line in file is used as Customer-Provided Encryption Keys for uploading and chnaging headers etc.
|
||||
if there are some keys after first line, those are used downloading object which are encripted by not first key.
|
||||
so that, you can keep all SSE-C keys in file, that is SSE-C key history.
|
||||
.TP
|
||||
\fB\-o\fR passwd_file (default="")
|
||||
specify the path to the password file, which which takes precedence over the password in $HOME/.passwd-s3fs and /etc/passwd-s3fs
|
||||
|
||||
340
src/curl.cpp
340
src/curl.cpp
@ -51,6 +51,61 @@
|
||||
|
||||
using namespace std;
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utilities
|
||||
//-------------------------------------------------------------------
|
||||
// [TODO]
|
||||
// This function uses tempolary file, but should not use it.
|
||||
// For not using it, we implement function in each auth file(openssl, nss. gnutls).
|
||||
//
|
||||
static bool make_md5_from_string(const char* pstr, string& md5)
|
||||
{
|
||||
if(!pstr || '\0' == pstr[0]){
|
||||
DPRN("Parameter is wrong.");
|
||||
return false;
|
||||
}
|
||||
FILE* fp;
|
||||
if(NULL == (fp = tmpfile())){
|
||||
FPRN("Could not make tmpfile.");
|
||||
return false;
|
||||
}
|
||||
size_t length = strlen(pstr);
|
||||
if(length != fwrite(pstr, sizeof(char), length, fp)){
|
||||
FPRN("Failed to write tmpfile.");
|
||||
fclose(fp);
|
||||
return false;
|
||||
}
|
||||
int fd;
|
||||
if(0 != fflush(fp) || 0 != fseek(fp, 0L, SEEK_SET) || -1 == (fd = fileno(fp))){
|
||||
FPRN("Failed to make MD5.");
|
||||
fclose(fp);
|
||||
return false;
|
||||
}
|
||||
// base64 md5
|
||||
md5 = s3fs_get_content_md5(fd);
|
||||
if(0 == md5.length()){
|
||||
FPRN("Failed to make MD5.");
|
||||
fclose(fp);
|
||||
return false;
|
||||
}
|
||||
fclose(fp);
|
||||
return true;
|
||||
}
|
||||
|
||||
static string tolower_header_name(const char* head)
|
||||
{
|
||||
string::size_type pos;
|
||||
string name = head;
|
||||
string value("");
|
||||
if(string::npos != (pos = name.find(':'))){
|
||||
value= name.substr(pos);
|
||||
name = name.substr(0, pos);
|
||||
}
|
||||
name = lower(name);
|
||||
name += value;
|
||||
return name;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Class BodyData
|
||||
//-------------------------------------------------------------------
|
||||
@ -154,6 +209,7 @@ int S3fsCurl::retries = 3; // default
|
||||
bool S3fsCurl::is_public_bucket = false;
|
||||
string S3fsCurl::default_acl = "private";
|
||||
bool S3fsCurl::is_use_rrs = false;
|
||||
sseckeylist_t S3fsCurl::sseckeys;
|
||||
bool S3fsCurl::is_use_sse = false;
|
||||
bool S3fsCurl::is_content_md5 = false;
|
||||
bool S3fsCurl::is_verbose = false;
|
||||
@ -684,6 +740,100 @@ bool S3fsCurl::SetUseRrs(bool flag)
|
||||
return old;
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetSseKeys(const char* filepath)
|
||||
{
|
||||
if(!filepath){
|
||||
DPRN("SSE-C keys filepath is empty.");
|
||||
return false;
|
||||
}
|
||||
S3fsCurl::sseckeys.clear();
|
||||
|
||||
ifstream ssefs(filepath);
|
||||
if(!ssefs.good()){
|
||||
FPRN("Could not open SSE-C keys file(%s).", filepath);
|
||||
return false;
|
||||
}
|
||||
|
||||
string line;
|
||||
while(getline(ssefs, line)){
|
||||
line = trim(line);
|
||||
if(0 == line.size()){
|
||||
continue;
|
||||
}
|
||||
if('#' == line[0]){
|
||||
continue;
|
||||
}
|
||||
// make base64
|
||||
char* pbase64_key;
|
||||
if(NULL == (pbase64_key = s3fs_base64((unsigned char*)line.c_str(), line.length()))){
|
||||
FPRN("Failed to convert base64 from sse-c key %s", line.c_str());
|
||||
continue;
|
||||
}
|
||||
string base64_key = pbase64_key;
|
||||
free(pbase64_key);
|
||||
|
||||
// make MD5
|
||||
string strMd5;
|
||||
if(!make_md5_from_string(line.c_str(), strMd5)){
|
||||
FPRN("Could not make MD5 from SSE-C keys(%s).", line.c_str());
|
||||
return false;
|
||||
}
|
||||
// mapped MD5 = SSE Key
|
||||
sseckeymap_t md5map;
|
||||
md5map.clear();
|
||||
md5map[strMd5] = base64_key;
|
||||
S3fsCurl::sseckeys.push_back(md5map);
|
||||
}
|
||||
if(0 == S3fsCurl::sseckeys.size()){
|
||||
FPRN("There is no SSE Key in file(%s).", filepath);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// If md5 is empty, returns first(current) sse key.
|
||||
//
|
||||
bool S3fsCurl::GetSseKey(string& md5, string& ssekey)
|
||||
{
|
||||
for(sseckeylist_t::const_iterator iter = S3fsCurl::sseckeys.begin(); iter != S3fsCurl::sseckeys.end(); iter++){
|
||||
if(0 == md5.length() || md5 == (*iter).begin()->first){
|
||||
md5 = iter->begin()->first;
|
||||
ssekey = iter->begin()->second;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool S3fsCurl::GetSseKeyMd5(int pos, string& md5)
|
||||
{
|
||||
if(pos < 0){
|
||||
return false;
|
||||
}
|
||||
if(S3fsCurl::sseckeys.size() <= static_cast<size_t>(pos)){
|
||||
return false;
|
||||
}
|
||||
int cnt = 0;
|
||||
for(sseckeylist_t::const_iterator iter = S3fsCurl::sseckeys.begin(); iter != S3fsCurl::sseckeys.end(); iter++, cnt++){
|
||||
if(pos == cnt){
|
||||
md5 = iter->begin()->first;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
int S3fsCurl::GetSseKeyCount(void)
|
||||
{
|
||||
return S3fsCurl::sseckeys.size();
|
||||
}
|
||||
|
||||
bool S3fsCurl::IsSseCustomMode(void)
|
||||
{
|
||||
return (0 < S3fsCurl::sseckeys.size());
|
||||
}
|
||||
|
||||
bool S3fsCurl::SetUseSse(bool flag)
|
||||
{
|
||||
bool old = S3fsCurl::is_use_sse;
|
||||
@ -911,7 +1061,7 @@ S3fsCurl* S3fsCurl::ParallelGetObjectRetryCallback(S3fsCurl* s3fscurl)
|
||||
// duplicate request(setup new curl object)
|
||||
S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
|
||||
if(0 != (result = newcurl->PreGetObjectRequest(
|
||||
s3fscurl->path.c_str(), s3fscurl->partdata.fd, s3fscurl->partdata.startpos, s3fscurl->partdata.size))){
|
||||
s3fscurl->path.c_str(), s3fscurl->partdata.fd, s3fscurl->partdata.startpos, s3fscurl->partdata.size, s3fscurl->b_ssekey_md5))){
|
||||
DPRN("failed downloading part setup(%d)", result);
|
||||
delete newcurl;
|
||||
return NULL;;
|
||||
@ -925,6 +1075,12 @@ int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, s
|
||||
{
|
||||
FPRNNN("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd);
|
||||
|
||||
string sseckeymd5("");
|
||||
char* psseckeymd5;
|
||||
if(NULL != (psseckeymd5 = get_object_sseckey_md5(tpath))){
|
||||
sseckeymd5 = psseckeymd5;
|
||||
free(psseckeymd5);
|
||||
}
|
||||
int result = 0;
|
||||
ssize_t remaining_bytes;
|
||||
|
||||
@ -945,7 +1101,7 @@ int S3fsCurl::ParallelGetObjectRequest(const char* tpath, int fd, off_t start, s
|
||||
|
||||
// s3fscurl sub object
|
||||
S3fsCurl* s3fscurl_para = new S3fsCurl();
|
||||
if(0 != (result = s3fscurl_para->PreGetObjectRequest(tpath, fd, (start + size - remaining_bytes), chunk))){
|
||||
if(0 != (result = s3fscurl_para->PreGetObjectRequest(tpath, fd, (start + size - remaining_bytes), chunk, sseckeymd5))){
|
||||
DPRN("failed downloading part setup(%d)", result);
|
||||
delete s3fscurl_para;
|
||||
return result;
|
||||
@ -1053,7 +1209,8 @@ bool S3fsCurl::CheckIAMCredentialUpdate(void)
|
||||
S3fsCurl::S3fsCurl(bool ahbe) :
|
||||
hCurl(NULL), path(""), base_path(""), saved_path(""), url(""), requestHeaders(NULL),
|
||||
bodydata(NULL), headdata(NULL), LastResponseCode(-1), postdata(NULL), postdata_remaining(0), is_use_ahbe(ahbe),
|
||||
retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0)
|
||||
retry_count(0), b_infile(NULL), b_postdata(NULL), b_postdata_remaining(0), b_partdata_startpos(0), b_partdata_size(0),
|
||||
b_ssekey_pos(-1), b_ssekey_md5("")
|
||||
{
|
||||
type = REQTYPE_UNSET;
|
||||
}
|
||||
@ -1728,12 +1885,38 @@ int S3fsCurl::GetIAMCredentials(void)
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// If md5 is empty, build by first(current) sse key
|
||||
//
|
||||
bool S3fsCurl::AddSseKeyRequestHead(string& md5, bool is_copy_source)
|
||||
{
|
||||
if(!S3fsCurl::IsSseCustomMode()){
|
||||
// Nothing to do
|
||||
return true;
|
||||
}
|
||||
string sseckey;
|
||||
if(S3fsCurl::GetSseKey(md5, sseckey)){
|
||||
if(is_copy_source){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-copy-source-server-side-encryption-customer-algorithm:AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-copy-source-server-side-encryption-customer-key:" + sseckey).c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-copy-source-server-side-encryption-customer-key-md5:" + md5).c_str());
|
||||
}else{
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption-customer-algorithm:AES256");
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-server-side-encryption-customer-key:" + sseckey).c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("x-amz-server-side-encryption-customer-key-md5:" + md5).c_str());
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//
|
||||
// tpath : target path for head request
|
||||
// bpath : saved into base_path
|
||||
// savedpath : saved into saved_path
|
||||
// ssekey_pos : -1 means "not use sse", 0 - X means "use sseckey" and "sseckey position".
|
||||
// sseckey position 0 is latest key.
|
||||
//
|
||||
bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* savedpath)
|
||||
bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char* savedpath, int ssekey_pos)
|
||||
{
|
||||
FPRNINFO("[tpath=%s][bpath=%s][save=%s]", SAFESTRPTR(tpath), SAFESTRPTR(bpath), SAFESTRPTR(savedpath));
|
||||
|
||||
@ -1759,6 +1942,15 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
|
||||
string date = get_date();
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string("Date: " + date).c_str());
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "Content-Type: ");
|
||||
|
||||
if(0 <= ssekey_pos && S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!S3fsCurl::GetSseKeyMd5(ssekey_pos, md5) || !AddSseKeyRequestHead(md5, false)){
|
||||
DPRN("Failed to set SSE-C headers for md5(%s).", md5.c_str());
|
||||
}
|
||||
}
|
||||
b_ssekey_pos = ssekey_pos;
|
||||
|
||||
if(!S3fsCurl::IsPublicBucket()){
|
||||
requestHeaders = curl_slist_sort_insert(
|
||||
requestHeaders,
|
||||
@ -1782,32 +1974,53 @@ bool S3fsCurl::PreHeadRequest(const char* tpath, const char* bpath, const char*
|
||||
|
||||
int S3fsCurl::HeadRequest(const char* tpath, headers_t& meta)
|
||||
{
|
||||
int result;
|
||||
int result = -1;
|
||||
|
||||
FPRNNN("[tpath=%s]", SAFESTRPTR(tpath));
|
||||
|
||||
if(!PreHeadRequest(tpath)){
|
||||
return -1;
|
||||
}
|
||||
// Requests
|
||||
if(0 != (result = RequestPerform())){
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
// SSE-C mode, check all sse-c key at first
|
||||
int pos;
|
||||
for(pos = 0; static_cast<size_t>(pos) < S3fsCurl::sseckeys.size(); pos++){
|
||||
if(0 != pos && !DestroyCurlHandle()){
|
||||
return result;
|
||||
}
|
||||
if(!PreHeadRequest(tpath, NULL, NULL, pos)){
|
||||
return result;
|
||||
}
|
||||
if(0 == (result = RequestPerform())){
|
||||
break;
|
||||
}
|
||||
}
|
||||
if(S3fsCurl::sseckeys.size() <= static_cast<size_t>(pos)){
|
||||
// If sse-c mode is enable, s3fs fails to get head request for normal and sse object.
|
||||
// So try to get head without sse-c header.
|
||||
if(!DestroyCurlHandle() || !PreHeadRequest(tpath, NULL, NULL, -1) || 0 != (result = RequestPerform())){
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
// Not sse-c mode
|
||||
if(!PreHeadRequest(tpath) || 0 != (result = RequestPerform())){
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
// file exists in s3
|
||||
// fixme: clean this up.
|
||||
meta.clear();
|
||||
for(headers_t::iterator iter = responseHeaders.begin(); iter != responseHeaders.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
if(key == "Content-Type"){
|
||||
if(0 == strcasecmp(key.c_str(), "Content-Type")){
|
||||
meta[key] = value;
|
||||
}else if(key == "Content-Length"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "Content-Length")){
|
||||
meta[key] = value;
|
||||
}else if(key == "ETag"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "ETag")){
|
||||
meta[key] = value;
|
||||
}else if(key == "Last-Modified"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "Last-Modified")){
|
||||
meta[key] = value;
|
||||
}else if(key.substr(0, 5) == "x-amz"){
|
||||
}else if(0 == strcasecmp(key.substr(0, 5).c_str(), "x-amz")){
|
||||
meta[key] = value;
|
||||
}else{
|
||||
// Check for upper case
|
||||
@ -1848,17 +2061,23 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
if(key == "Content-Type"){
|
||||
if(0 == strcasecmp(key.c_str(), "Content-Type")){
|
||||
ContentType = value;
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key.substr(0,9) == "x-amz-acl"){
|
||||
}else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
|
||||
// not set value, but after set it.
|
||||
}else if(key.substr(0,10) == "x-amz-meta"){
|
||||
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key == "x-amz-copy-source"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(!ow_sse_flg && key == "x-amz-server-side-encryption"){
|
||||
// If ow_sse_flg is false, SSE inherit from meta.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Not need to check error.
|
||||
if(!AddSseKeyRequestHead(value, ow_sse_flg)){ // ow_sse_flg=true means copy source
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}else if(!ow_sse_flg && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}
|
||||
}
|
||||
@ -1867,8 +2086,15 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg
|
||||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg && S3fsCurl::is_use_sse){
|
||||
if(ow_sse_flg){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
@ -1953,15 +2179,21 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
if(key == "Content-Type"){
|
||||
if(0 == strcasecmp(key.c_str(), "Content-Type")){
|
||||
ContentType = value;
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key.substr(0,9) == "x-amz-acl"){
|
||||
}else if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
|
||||
// not set value, but after set it.
|
||||
}else if(key.substr(0,10) == "x-amz-meta"){
|
||||
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(!ow_sse_flg && key == "x-amz-server-side-encryption"){
|
||||
// If ow_sse_flg is false, SSE inherit from meta.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Not need to check error.
|
||||
if(!AddSseKeyRequestHead(value, ow_sse_flg)){ // ow_sse_flg=true means copy source
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}else if(!ow_sse_flg && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}
|
||||
}
|
||||
@ -1970,8 +2202,15 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
||||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg && S3fsCurl::is_use_sse){
|
||||
if(ow_sse_flg){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
@ -2011,7 +2250,7 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse
|
||||
return result;
|
||||
}
|
||||
|
||||
int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size)
|
||||
int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, string& ssekeymd5)
|
||||
{
|
||||
FPRNNN("[tpath=%s][start=%jd][size=%zd]", SAFESTRPTR(tpath), (intmax_t)start, size);
|
||||
|
||||
@ -2041,6 +2280,11 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
|
||||
range += str(start + size - 1);
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, range.c_str());
|
||||
}
|
||||
if(0 < ssekeymd5.length()){
|
||||
if(!AddSseKeyRequestHead(ssekeymd5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
|
||||
if(!S3fsCurl::IsPublicBucket()){
|
||||
requestHeaders = curl_slist_sort_insert(
|
||||
@ -2063,6 +2307,7 @@ int S3fsCurl::PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_
|
||||
partdata.size = size;
|
||||
b_partdata_startpos = start;
|
||||
b_partdata_size = size;
|
||||
b_ssekey_md5 = ssekeymd5;
|
||||
|
||||
type = REQTYPE_GET;
|
||||
|
||||
@ -2078,7 +2323,13 @@ int S3fsCurl::GetObjectRequest(const char* tpath, int fd, off_t start, ssize_t s
|
||||
if(!tpath){
|
||||
return -1;
|
||||
}
|
||||
if(0 != (result = PreGetObjectRequest(tpath, fd, start, size))){
|
||||
string sseckeymd5("");
|
||||
char* psseckeymd5;
|
||||
if(NULL != (psseckeymd5 = get_object_sseckey_md5(tpath))){
|
||||
sseckeymd5 = psseckeymd5;
|
||||
free(psseckeymd5);
|
||||
}
|
||||
if(0 != (result = PreGetObjectRequest(tpath, fd, start, size, sseckeymd5))){
|
||||
return result;
|
||||
}
|
||||
|
||||
@ -2220,12 +2471,18 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
|
||||
if(key.substr(0,9) == "x-amz-acl"){
|
||||
if(0 == strcasecmp(key.substr(0,9).c_str(), "x-amz-acl")){
|
||||
// not set value, but after set it.
|
||||
}else if(key.substr(0,10) == "x-amz-meta"){
|
||||
}else if(0 == strcasecmp(key.substr(0,10).c_str(), "x-amz-meta")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(!ow_sse_flg && key == "x-amz-server-side-encryption"){
|
||||
// If ow_sse_flg is false, SSE inherit from meta.
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-algorithm")){
|
||||
// skip this header, because this header is specified with "x-amz-...-customer-key-md5".
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
// Not need to check error.
|
||||
if(!AddSseKeyRequestHead(value, ow_sse_flg)){ // ow_sse_flg=true means copy source
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}else if(!ow_sse_flg && 0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}
|
||||
}
|
||||
@ -2234,8 +2491,15 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
|
||||
if(S3fsCurl::is_use_rrs){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class:REDUCED_REDUNDANCY");
|
||||
}
|
||||
if(ow_sse_flg && S3fsCurl::is_use_sse){
|
||||
if(ow_sse_flg){
|
||||
if(S3fsCurl::is_use_sse){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption:AES256");
|
||||
}else if(S3fsCurl::IsSseCustomMode()){
|
||||
string md5;
|
||||
if(!AddSseKeyRequestHead(md5, false)){
|
||||
DPRNNN("Failed to insert sse(-c) header.");
|
||||
}
|
||||
}
|
||||
}
|
||||
if(is_use_ahbe){
|
||||
// set additional header by ahbe conf
|
||||
@ -2584,12 +2848,12 @@ int S3fsCurl::CopyMultipartPostRequest(const char* from, const char* to, int par
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
string value = (*iter).second;
|
||||
if(key == "Content-Type"){
|
||||
if(0 == strcasecmp(key.c_str(), "Content-Type")){
|
||||
ContentType = value;
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key == "x-amz-copy-source"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}else if(key == "x-amz-copy-source-range"){
|
||||
}else if(0 == strcasecmp(key.c_str(), "x-amz-copy-source-range")){
|
||||
requestHeaders = curl_slist_sort_insert(requestHeaders, string(key + ":" + value).c_str());
|
||||
}
|
||||
// NOTICE: x-amz-acl, x-amz-server-side-encryption is not set!
|
||||
|
||||
20
src/curl.h
20
src/curl.h
@ -100,6 +100,8 @@ class S3fsMultiCurl;
|
||||
// class S3fsCurl
|
||||
//----------------------------------------------
|
||||
typedef std::map<std::string, std::string> iamcredmap_t;
|
||||
typedef std::map<std::string, std::string> sseckeymap_t;
|
||||
typedef std::list<sseckeymap_t> sseckeylist_t;
|
||||
|
||||
// share
|
||||
#define SHARE_MUTEX_DNS 0
|
||||
@ -144,6 +146,7 @@ class S3fsCurl
|
||||
static bool is_public_bucket;
|
||||
static std::string default_acl; // TODO: to enum
|
||||
static bool is_use_rrs;
|
||||
static sseckeylist_t sseckeys;
|
||||
static bool is_use_sse;
|
||||
static bool is_content_md5;
|
||||
static bool is_verbose;
|
||||
@ -182,6 +185,8 @@ class S3fsCurl
|
||||
int b_postdata_remaining; // backup for retrying
|
||||
off_t b_partdata_startpos; // backup for retrying
|
||||
ssize_t b_partdata_size; // backup for retrying
|
||||
bool b_ssekey_pos; // backup for retrying
|
||||
std::string b_ssekey_md5; // backup for retrying
|
||||
|
||||
public:
|
||||
// constructor/destructor
|
||||
@ -250,6 +255,11 @@ class S3fsCurl
|
||||
static std::string SetDefaultAcl(const char* acl);
|
||||
static bool SetUseRrs(bool flag);
|
||||
static bool GetUseRrs(void) { return S3fsCurl::is_use_rrs; }
|
||||
static bool SetSseKeys(const char* filepath);
|
||||
static bool GetSseKey(std::string& md5, std::string& ssekey);
|
||||
static bool GetSseKeyMd5(int pos, std::string& md5);
|
||||
static int GetSseKeyCount(void);
|
||||
static bool IsSseCustomMode(void);
|
||||
static bool SetUseSse(bool flag);
|
||||
static bool GetUseSse(void) { return S3fsCurl::is_use_sse; }
|
||||
static bool SetContentMd5(bool flag);
|
||||
@ -272,17 +282,18 @@ class S3fsCurl
|
||||
bool CreateCurlHandle(bool force = false);
|
||||
bool DestroyCurlHandle(void);
|
||||
|
||||
bool AddSseKeyRequestHead(std::string& md5, bool is_copy_source);
|
||||
bool GetResponseCode(long& responseCode);
|
||||
int RequestPerform(void);
|
||||
int DeleteRequest(const char* tpath);
|
||||
bool PreHeadRequest(const char* tpath, const char* bpath = NULL, const char* savedpath = NULL);
|
||||
bool PreHeadRequest(std::string& tpath, std::string& bpath, std::string& savedpath) {
|
||||
return PreHeadRequest(tpath.c_str(), bpath.c_str(), savedpath.c_str());
|
||||
bool PreHeadRequest(const char* tpath, const char* bpath = NULL, const char* savedpath = NULL, int ssekey_pos = -1);
|
||||
bool PreHeadRequest(std::string& tpath, std::string& bpath, std::string& savedpath, int ssekey_pos = -1) {
|
||||
return PreHeadRequest(tpath.c_str(), bpath.c_str(), savedpath.c_str(), ssekey_pos);
|
||||
}
|
||||
int HeadRequest(const char* tpath, headers_t& meta);
|
||||
int PutHeadRequest(const char* tpath, headers_t& meta, bool ow_sse_flg);
|
||||
int PutRequest(const char* tpath, headers_t& meta, int fd, bool ow_sse_flg);
|
||||
int PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size);
|
||||
int PreGetObjectRequest(const char* tpath, int fd, off_t start, ssize_t size, std::string& ssekeymd5);
|
||||
int GetObjectRequest(const char* tpath, int fd, off_t start = -1, ssize_t size = -1);
|
||||
int CheckBucket(void);
|
||||
int ListBucketRequest(const char* tpath, const char* query);
|
||||
@ -309,6 +320,7 @@ class S3fsCurl
|
||||
int GetMultipartRetryCount(void) const { return retry_count; }
|
||||
void SetMultipartRetryCount(int retrycnt) { retry_count = retrycnt; }
|
||||
bool IsOverMultipartRetryCount(void) const { return (retry_count >= S3fsCurl::retries); }
|
||||
int GetLastPreHeadSeecKeyPos(void) const { return b_ssekey_pos; }
|
||||
};
|
||||
|
||||
//----------------------------------------------
|
||||
|
||||
76
src/s3fs.cpp
76
src/s3fs.cpp
@ -610,6 +610,30 @@ static int check_parent_object_access(const char* path, int mask)
|
||||
return 0;
|
||||
}
|
||||
|
||||
//
|
||||
// This function is global, is called fom curl class(GetObject).
|
||||
//
|
||||
char* get_object_sseckey_md5(const char* path)
|
||||
{
|
||||
if(!path){
|
||||
return NULL;
|
||||
}
|
||||
headers_t meta;
|
||||
|
||||
if(0 != get_object_attribute(path, NULL, &meta)){
|
||||
DPRNNN("Failed to get object(%s) headers", path);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
|
||||
string key = (*iter).first;
|
||||
if(0 == strcasecmp(key.c_str(), "x-amz-server-side-encryption-customer-key-md5")){
|
||||
return strdup((*iter).second.c_str());
|
||||
}
|
||||
}
|
||||
return NULL;
|
||||
}
|
||||
|
||||
static FdEntity* get_local_fent(const char* path, bool is_load)
|
||||
{
|
||||
struct stat stobj;
|
||||
@ -2078,22 +2102,36 @@ static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl)
|
||||
if(!s3fscurl){
|
||||
return NULL;
|
||||
}
|
||||
int ssec_key_pos = s3fscurl->GetLastPreHeadSeecKeyPos();
|
||||
int next_retry_count = s3fscurl->GetMultipartRetryCount() + 1;
|
||||
|
||||
if(s3fscurl->IsOverMultipartRetryCount()){
|
||||
if(S3fsCurl::IsSseCustomMode()){
|
||||
// If sse-c mode, start check not sse-c(ssec_key_pos = -1).
|
||||
// do increment ssec_key_pos for checking all sse-c key.
|
||||
next_retry_count = 0;
|
||||
ssec_key_pos++;
|
||||
if(S3fsCurl::GetSseKeyCount() <= ssec_key_pos){
|
||||
DPRN("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpacialSavedPath().c_str());
|
||||
return NULL;
|
||||
}
|
||||
}else{
|
||||
DPRN("Over retry count(%d) limit(%s).", s3fscurl->GetMultipartRetryCount(), s3fscurl->GetSpacialSavedPath().c_str());
|
||||
return NULL;
|
||||
}
|
||||
}
|
||||
|
||||
S3fsCurl* newcurl = new S3fsCurl(s3fscurl->IsUseAhbe());
|
||||
string path = s3fscurl->GetPath();
|
||||
string base_path = s3fscurl->GetBasePath();
|
||||
string saved_path = s3fscurl->GetSpacialSavedPath();
|
||||
|
||||
if(!newcurl->PreHeadRequest(path, base_path, saved_path)){
|
||||
if(!newcurl->PreHeadRequest(path, base_path, saved_path, ssec_key_pos)){
|
||||
DPRN("Could not duplicate curl object(%s).", saved_path.c_str());
|
||||
delete newcurl;
|
||||
return NULL;
|
||||
}
|
||||
newcurl->SetMultipartRetryCount(s3fscurl->GetMultipartRetryCount() + 1);
|
||||
newcurl->SetMultipartRetryCount(next_retry_count);
|
||||
|
||||
return newcurl;
|
||||
}
|
||||
@ -2135,6 +2173,8 @@ static int readdir_multi_head(const char* path, S3ObjList& head, void* buf, fuse
|
||||
continue;
|
||||
}
|
||||
|
||||
// First check for directory, start checking "not sse-c".
|
||||
// If checking failed, retry to check with "sse-c" by retry callback func when sse-c mode.
|
||||
S3fsCurl* s3fscurl = new S3fsCurl();
|
||||
if(!s3fscurl->PreHeadRequest(disppath, (*iter), disppath)){ // target path = cache key path.(ex "dir/")
|
||||
DPRNNN("Could not make curl object for head request(%s).", disppath.c_str());
|
||||
@ -3522,22 +3562,36 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == strcmp(arg, "use_sse") || 0 == STR2NCMP(arg, "use_sse=")){
|
||||
off_t sse = 1;
|
||||
// for an old format.
|
||||
if(0 == STR2NCMP(arg, "use_sse=")){
|
||||
sse = s3fs_strtoofft(strchr(arg, '=') + sizeof(char));
|
||||
if(S3fsCurl::GetUseRrs()){
|
||||
fprintf(stderr, "%s: use_sse option could not be specified with use_rrs.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
if(0 == sse){
|
||||
S3fsCurl::SetUseSse(false);
|
||||
}else if(1 == sse){
|
||||
const char* ssecfile = &arg[strlen("use_sse=")];
|
||||
if(0 == strcmp(ssecfile, "1")){
|
||||
S3fsCurl::SetUseSse(true);
|
||||
}else{
|
||||
// testing sse-c, try to load AES256 keys
|
||||
struct stat st;
|
||||
if(0 != stat(ssecfile, &st)){
|
||||
fprintf (stderr, "%s: could not open use_sse keys file(%s)\n", program_name.c_str(), ssecfile);
|
||||
return -1;
|
||||
}
|
||||
if(st.st_mode & (S_IXUSR | S_IRWXG | S_IRWXO)){
|
||||
fprintf (stderr, "%s: use_sse keys file %s should be 0600 permissions\n", program_name.c_str(), ssecfile);
|
||||
return -1;
|
||||
}
|
||||
if(!S3fsCurl::SetSseKeys(ssecfile)){
|
||||
fprintf (stderr, "%s: failed to load use_sse keys file %s\n", program_name.c_str(), ssecfile);
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(S3fsCurl::GetUseRrs()){
|
||||
fprintf(stderr, "%s: use_sse option could not be specified with use_rrs.\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
S3fsCurl::SetUseSse(true);
|
||||
}else{
|
||||
fprintf(stderr, "%s: poorly formed argument to option: use_sse\n", program_name.c_str());
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
@ -65,4 +65,6 @@
|
||||
|
||||
#endif // HAVE_MALLOC_TRIM
|
||||
|
||||
char* get_object_sseckey_md5(const char* path);
|
||||
|
||||
#endif // S3FS_S3_H_
|
||||
|
||||
@ -875,7 +875,22 @@ void show_help (void)
|
||||
" - this option makes Amazon's Reduced Redundancy Storage enable.\n"
|
||||
"\n"
|
||||
" use_sse (default is disable)\n"
|
||||
" - this option makes Amazon's Server Site Encryption enable.\n"
|
||||
" - use Amazon<6F>fs Server-Site Encryption or Server-Side Encryption\n"
|
||||
" with Customer-Provided Encryption Keys.\n"
|
||||
" this option can not be specified with use_rrs. specifying only \n"
|
||||
" \"use_sse\" or \"use_sse=1\" enables Server-Side Encryption.\n"
|
||||
" (use_sse=1 for old version)\n"
|
||||
" specifying this option with file path which has some SSE-C\n"
|
||||
" secret key enables Server-Side Encryption with Customer-Provided\n"
|
||||
" Encryption Keys.(use_sse=file)\n"
|
||||
" the file must be 600 permission. the file can have some lines,\n"
|
||||
" each line is one SSE-C key. the first line in file is used as\n"
|
||||
" Customer-Provided Encryption Keys for uploading and chnaging\n"
|
||||
" headers etc.\n"
|
||||
" if there are some keys after first line, those are used\n"
|
||||
" downloading object which are encripted by not first key.\n"
|
||||
" so that, you can keep all SSE-C keys in file, that is SSE-C\n"
|
||||
" key history.\n"
|
||||
"\n"
|
||||
" public_bucket (default=\"\" which means disabled)\n"
|
||||
" - anonymously mount a public bucket when set to 1\n"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user