octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-23 09:18:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add support for standard_ia storage class

Browse Source 
This enables storage with lower at-rest prices, higher request prices,
and lower availability.  Also rework existing reduced redundancy
parsing into a more generic storage class.  More background on
standard_ia:

https://aws.amazon.com/blogs/aws/aws-storage-update-new-lower-cost-s3-storage-option-glacier-price-reduction/
This commit is contained in:
Andrew Gaul 2015-09-17 13:10:45 -07:00
parent 489f9edec7
commit 785ed642ba
4 changed files with 53 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
src/curl.cpp
View File

@ -252,7 +252,7 @@ time_t S3fsCurl::readwrite_timeout = 60; // default
int S3fsCurl::retries = 3; // default
bool S3fsCurl::is_public_bucket = false;
string S3fsCurl::default_acl = "private";
bool S3fsCurl::is_use_rrs = false;
storage_class_t S3fsCurl::storage_class = STANDARD;
sseckeylist_t S3fsCurl::sseckeys;
bool S3fsCurl::is_use_sse = false;
bool S3fsCurl::is_content_md5 = false;
@ -790,10 +790,10 @@ string S3fsCurl::SetDefaultAcl(const char* acl)
return old;
}
bool S3fsCurl::SetUseRrs(bool flag)
storage_class_t S3fsCurl::SetStorageClass(storage_class_t storage_class)
{
bool old = S3fsCurl::is_use_rrs;
S3fsCurl::is_use_rrs = flag;
storage_class_t old = S3fsCurl::storage_class;
S3fsCurl::storage_class = storage_class;
return old;
}
@ -2263,10 +2263,12 @@ int S3fsCurl::PutHeadRequest(const char* tpath, headers_t& meta, bool is_copy)
}
}
// "x-amz-acl", rrs, sse
// "x-amz-acl", storage class, sse
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.c_str());
if(S3fsCurl::is_use_rrs){
if(REDUCED_REDUNDANCY == GetStorageClass()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "REDUCED_REDUNDANCY");
} else if(STANDARD_IA == GetStorageClass()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "STANDARD_IA");
}
if(S3fsCurl::is_use_sse){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
@ -2381,10 +2383,12 @@ int S3fsCurl::PutRequest(const char* tpath, headers_t& meta, int fd)
// skip this header, because this header is specified after logic.
}
}
// "x-amz-acl", rrs, sse
// "x-amz-acl", storage class, sse
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.c_str());
if(S3fsCurl::is_use_rrs){
if(REDUCED_REDUNDANCY == GetStorageClass()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "REDUCED_REDUNDANCY");
} else if(STANDARD_IA == GetStorageClass()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "STANDARD_IA");
}
if(S3fsCurl::is_use_sse){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");
@ -2684,10 +2688,12 @@ int S3fsCurl::PreMultipartPostRequest(const char* tpath, headers_t& meta, string
}
}
}
// "x-amz-acl", rrs, sse
// "x-amz-acl", storage class, sse
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-acl", S3fsCurl::default_acl.c_str());
if(S3fsCurl::is_use_rrs){
if(REDUCED_REDUNDANCY == GetStorageClass()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "REDUCED_REDUNDANCY");
} else if(STANDARD_IA == GetStorageClass()){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-storage-class", "STANDARD_IA");
}
if(S3fsCurl::is_use_sse){
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-server-side-encryption", "AES256");

12
src/curl.h
View File

@ -122,6 +122,12 @@ typedef std::map<std::string, std::string> iamcredmap_t;
typedef std::map<std::string, std::string> sseckeymap_t;
typedef std::list<sseckeymap_t> sseckeylist_t;
enum storage_class_t {
STANDARD,
STANDARD_IA,
REDUCED_REDUNDANCY,
};
// share
#define SHARE_MUTEX_DNS 0
#define SHARE_MUTEX_SSL_SESSION 1
@ -165,7 +171,7 @@ class S3fsCurl
static int retries;
static bool is_public_bucket;
static std::string default_acl; // TODO: to enum
static bool is_use_rrs;
static storage_class_t storage_class;
static sseckeylist_t sseckeys;
static bool is_use_sse;
static bool is_content_md5;
@ -278,8 +284,8 @@ class S3fsCurl
static bool SetPublicBucket(bool flag);
static bool IsPublicBucket(void) { return S3fsCurl::is_public_bucket; }
static std::string SetDefaultAcl(const char* acl);
static bool SetUseRrs(bool flag);
static bool GetUseRrs(void) { return S3fsCurl::is_use_rrs; }
static storage_class_t SetStorageClass(storage_class_t storage_class);
static storage_class_t GetStorageClass() { return S3fsCurl::storage_class; }
static bool SetSseKeys(const char* filepath);
static bool LoadEnvSseKeys(void);
static bool GetSseKey(std::string& md5, std::string& ssekey);

31
src/s3fs.cpp
View File

@ -4218,6 +4218,7 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
nomultipart = true;
return 0;
}
// old format for storage_class
if(0 == strcmp(arg, "use_rrs") || 0 == STR2NCMP(arg, "use_rrs=")){
off_t rrs = 1;
// for an old format.
@ -4225,23 +4226,41 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
rrs = s3fs_strtoofft(strchr(arg, '=') + sizeof(char));
}
if(0 == rrs){
S3fsCurl::SetUseRrs(false);
S3fsCurl::SetStorageClass(STANDARD);
}else if(1 == rrs){
if(S3fsCurl::GetUseSse()){
fprintf(stderr, "%s: use_rrs option could not be specified with use_sse.\n", program_name.c_str());
return -1;
}
S3fsCurl::SetUseRrs(true);
S3fsCurl::SetStorageClass(REDUCED_REDUNDANCY);
}else{
fprintf(stderr, "%s: poorly formed argument to option: use_rrs\n", program_name.c_str());
return -1;
}
return 0;
}
if(0 == STR2NCMP(arg, "storage_class=")){
const char *storage_class = strchr(arg, '=') + sizeof(char);
if(0 == strcmp(storage_class, "standard")){
S3fsCurl::SetStorageClass(STANDARD);
}else if(0 == strcmp(storage_class, "standard_ia")){
S3fsCurl::SetStorageClass(STANDARD_IA);
}else if(0 == strcmp(storage_class, "reduced_redundancy")){
if(S3fsCurl::GetUseSse()){
fprintf(stderr, "%s: storage class reduced_redundancy option could not be specified with use_sse.\n", program_name.c_str());
return -1;
}
S3fsCurl::SetStorageClass(REDUCED_REDUNDANCY);
}else{
fprintf(stderr, "%s: unknown value for storage_class: %s\n", program_name.c_str(), storage_class);
return -1;
}
return 0;
}
if(0 == strcmp(arg, "use_sse") || 0 == STR2NCMP(arg, "use_sse=")){
if(0 == STR2NCMP(arg, "use_sse=")){
if(S3fsCurl::GetUseRrs()){
fprintf(stderr, "%s: use_sse option could not be specified with use_rrs.\n", program_name.c_str());
if(REDUCED_REDUNDANCY == S3fsCurl::GetStorageClass()){
fprintf(stderr, "%s: use_sse option could not be specified with storage class reduced_redundancy.\n", program_name.c_str());
return -1;
}
const char* ssecfile = &arg[strlen("use_sse=")];
@ -4268,8 +4287,8 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
}
}
}else{
if(S3fsCurl::GetUseRrs()){
fprintf(stderr, "%s: use_sse option could not be specified with use_rrs.\n", program_name.c_str());
if(REDUCED_REDUNDANCY == S3fsCurl::GetStorageClass()){
fprintf(stderr, "%s: use_sse option could not be specified with storage class reduced_redundancy.\n", program_name.c_str());
return -1;
}
if(S3fsCurl::IsSseCustomMode()){

5
src/s3fs_util.cpp
View File

@ -894,8 +894,9 @@ void show_help (void)
" del_cache (delete local file cache)\n"
" - delete local file cache when s3fs starts and exits.\n"
"\n"
" use_rrs (default is disable)\n"
" - this option makes Amazon's Reduced Redundancy Storage enable.\n"
" storage_class (default=\"standard\")\n"
" - store object with specified storage class. Possible values:\n"
" standard, standard_ia, and reduced_redundancy.\n"
"\n"
" use_sse (default is disable)\n"
" - use Amazon's Server-Site Encryption or Server-Side Encryption\n"