octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-12-22 08:48:55 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #116 from s3fs-fuse/dev_sv4

Browse Source 
Supported signature version 4
This commit is contained in:
Takeshi Nakatani 2015-02-03 01:43:48 +09:00
commit 4ae5043534
14 changed files with 1260 additions and 265 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
doc/man/s3fs.1
View File

@ -149,6 +149,16 @@ This option is lated to fd_page_size option and affects it.
\fB\-o\fR url (default="http://s3.amazonaws.com")
sets the url to use to access Amazon S3. If you want to use HTTPS, then you can set url=https://s3.amazonaws.com
.TP
\fB\-o\fR endpoint (default="us-east-1")
sets the endpoint to use.
If this option is not specified, s3fs uses \"us-east-1\" region as the default.
If the s3fs could not connect to the region specified by this option, s3fs could not run.
But if you do not specify this option, and if you can not connect with the default region, s3fs will retry to automatically connect to the other region.
So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server.
.TP
\fB\-o\fR sigv2 (default is signature version 4)
sets signing AWS requests by sing Signature Version 2.
.TP
\fB\-o\fR nomultipart - disable multipart uploads
.TP
\fB\-o\fR enable_content_md5 ( default is disable )

29
src/cache.cpp
View File

@ -35,6 +35,7 @@
#include "cache.h"
#include "s3fs.h"
#include "s3fs_util.h"
#include "string_util.h"
using namespace std;
@ -269,24 +270,18 @@ bool StatCache::AddStat(std::string& key, headers_t& meta, bool forcedir)
ent->meta.clear();
//copy only some keys
for(headers_t::iterator iter = meta.begin(); iter != meta.end(); ++iter){
string tag = (*iter).first;
string value = (*iter).second;
if(tag == "Content-Type"){
ent->meta[tag] = value;
}else if(tag == "Content-Length"){
ent->meta[tag] = value;
}else if(tag == "ETag"){
ent->meta[tag] = value;
}else if(tag == "Last-Modified"){
ent->meta[tag] = value;
string tag = lower(iter->first);
string value = iter->second;
if(tag == "content-type"){
ent->meta[iter->first] = value;
}else if(tag == "content-length"){
ent->meta[iter->first] = value;
}else if(tag == "etag"){
ent->meta[iter->first] = value;
}else if(tag == "last-modified"){
ent->meta[iter->first] = value;
}else if(tag.substr(0, 5) == "x-amz"){
ent->meta[tag] = value;
}else{
// Check for upper case
transform(tag.begin(), tag.end(), tag.begin(), static_cast<int (*)(int)>(std::tolower));
if(tag.substr(0, 5) == "x-amz"){
ent->meta[tag] = value;
}
ent->meta[tag] = value; // key is lower case for "x-amz"
}
}
// add

1
src/common.h
View File

@ -94,6 +94,7 @@ extern std::string service_path;
extern std::string host;
extern std::string bucket;
extern std::string mount_prefix;
extern std::string endpoint;
#endif // S3FS_COMMON_H_

21
src/common_auth.cpp
View File

@ -102,6 +102,27 @@ string s3fs_md5sum(int fd, off_t start, ssize_t size)
return string(md5);
}
string s3fs_sha256sum(int fd, off_t start, ssize_t size)
{
size_t digestlen = get_sha256_digest_length();
char sha256[2 * digestlen + 1];
char hexbuf[3];
unsigned char* sha256hex;
if(NULL == (sha256hex = s3fs_sha256hexsum(fd, start, size))){
return string("");
}
memset(sha256, 0, 2 * digestlen + 1);
for(size_t pos = 0; pos < digestlen; pos++){
snprintf(hexbuf, 3, "%02x", sha256hex[pos]);
strncat(sha256, hexbuf, 2);
}
free(sha256hex);
return string(sha256);
}
/*
* Local variables:
* tab-width: 4

871
src/curl.cpp
View File

File diff suppressed because it is too large Load Diff

9
src/curl.h
View File

@ -181,6 +181,7 @@ class S3fsCurl
static mimes_t mimeTypes;
static int max_parallel_cnt;
static off_t multipart_size;
static bool is_sigv4;
// variables
CURL* hCurl;
@ -244,7 +245,8 @@ class S3fsCurl
bool ResetHandle(void);
bool RemakeHandle(void);
bool ClearInternalData(void);
std::string CalcSignature(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource);
std::string CalcSignatureV2(std::string method, std::string strMD5, std::string content_type, std::string date, std::string resource);
std::string CalcSignature(std::string method, std::string canonical_uri, std::string query_string, std::string strdate, std::string payload_hash, std::string date8601);
bool GetUploadId(std::string& upload_id);
int GetIAMCredentials(void);
@ -298,6 +300,8 @@ class S3fsCurl
static const char* GetIAMRole(void) { return S3fsCurl::IAM_role.c_str(); }
static bool SetMultipartSize(off_t size);
static off_t GetMultipartSize(void) { return S3fsCurl::multipart_size; }
static bool SetSignatureV4(bool isset = true) { bool bresult = S3fsCurl::is_sigv4; S3fsCurl::is_sigv4 = isset; return bresult; }
static bool IsSignatureV4(void) { return S3fsCurl::is_sigv4; }
// methods
bool CreateCurlHandle(bool force = false);
@ -421,6 +425,9 @@ std::string GetContentMD5(int fd);
unsigned char* md5hexsum(int fd, off_t start, ssize_t size);
std::string md5sum(int fd, off_t start, ssize_t size);
struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* data);
struct curl_slist* curl_slist_sort_insert(struct curl_slist* list, const char* key, const char* value);
std::string get_sorted_header_keys(const struct curl_slist* list);
std::string get_canonical_headers(const struct curl_slist* list, bool only_amz = false);
bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::string& url);
std::string prepare_url(const char* url);

187
src/gnutls_auth.cpp
View File

@ -120,6 +120,25 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
return true;
}
bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
{
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
return false;
}
if(NULL == (*digest = (unsigned char*)malloc(SHA256_DIGEST_SIZE))){
return false;
}
struct hmac_sha256_ctx ctx_hmac;
hmac_sha256_set_key(&ctx_hmac, keylen, reinterpret_cast<const uint8_t*>(key));
hmac_sha256_update(&ctx_hmac, datalen, reinterpret_cast<const uint8_t*>(data));
hmac_sha256_digest(&ctx_hmac, SHA256_DIGEST_SIZE, reinterpret_cast<uint8_t*>(*digest));
*digestlen = SHA256_DIGEST_SIZE;
return true;
}
#else // USE_GNUTLS_NETTLE
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
@ -142,6 +161,26 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
return true;
}
bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
{
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
return false;
}
if(0 >= (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA256))){
return false;
}
if(NULL == (*digest = (unsigned char*)malloc(*digestlen + 1))){
return false;
}
if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA256, key, keylen, data, datalen, *digest)){
free(*digest);
*digest = NULL;
return false;
}
return true;
}
#endif // USE_GNUTLS_NETTLE
//-------------------------------------------------------------------
@ -256,6 +295,154 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
#endif // USE_GNUTLS_NETTLE
//-------------------------------------------------------------------
// Utility Function for SHA256
//-------------------------------------------------------------------
#define SHA256_DIGEST_LENGTH 32
size_t get_sha256_digest_length(void)
{
return SHA256_DIGEST_LENGTH;
}
#ifdef USE_GNUTLS_NETTLE
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
{
(*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
return false;
}
struct sha256_ctx ctx_sha256;
sha256_init(&ctx_sha256);
sha256_update(&ctx_sha256, datalen, data);
sha256_digest(&ctx_sha256, *digestlen, *digest);
return true;
}
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
{
struct sha256_ctx ctx_sha256;
unsigned char buf[512];
ssize_t bytes;
unsigned char* result;
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512);
sha256_init(&ctx_sha256);
for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes);
if(0 == bytes){
// end of file
break;
}else if(-1 == bytes){
// error
DPRNNN("file read error(%d)", errno);
return NULL;
}
sha256_update(&ctx_sha256, bytes, buf);
memset(buf, 0, 512);
}
if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
return NULL;
}
sha256_digest(&ctx_sha256, get_sha256_digest_length(), result);
if(-1 == lseek(fd, start, SEEK_SET)){
free(result);
return NULL;
}
return result;
}
#else // USE_GNUTLS_NETTLE
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
{
(*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
return false;
}
gcry_md_hd_t ctx_sha256;
gcry_error_t err;
if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){
DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
free(*digest);
return false;
}
gcry_md_write(ctx_sha256, data, datalen);
memcpy(*digest, gcry_md_read(ctx_sha256, 0), *digestlen);
gcry_md_close(ctx_sha256);
return true;
}
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
{
gcry_md_hd_t ctx_sha256;
gcry_error_t err;
char buf[512];
ssize_t bytes;
unsigned char* result;
if(-1 == size){
struct stat st;
if(-1 == fstat(fd, &st)){
return NULL;
}
size = static_cast<ssize_t>(st.st_size);
}
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512);
if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){
DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
return NULL;
}
for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes);
if(0 == bytes){
// end of file
break;
}else if(-1 == bytes){
// error
DPRNNN("file read error(%d)", errno);
return NULL;
}
gcry_md_write(ctx_sha256, buf, bytes);
memset(buf, 0, 512);
}
if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
return NULL;
}
memcpy(result, gcry_md_read(ctx_sha256, 0), get_sha256_digest_length());
gcry_md_close(ctx_sha256);
if(-1 == lseek(fd, start, SEEK_SET)){
free(result);
return NULL;
}
return result;
}
#endif // USE_GNUTLS_NETTLE
/*
* Local variables:
* tab-width: 4

98
src/nss_auth.cpp
View File

@ -83,7 +83,7 @@ bool s3fs_destroy_crypt_mutex(void)
//-------------------------------------------------------------------
// Utility Function for HMAC
//-------------------------------------------------------------------
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen, bool is_sha256)
{
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
return false;
@ -94,17 +94,17 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
PK11Context* Context;
SECStatus SecStatus;
unsigned char tmpdigest[64];
SECItem KeySecItem = {siBuffer, reinterpret_cast<unsigned char*>(const_cast<void*>(key)), keylen};
SECItem KeySecItem = {siBuffer, reinterpret_cast<unsigned char*>(const_cast<void*>(key)), static_cast<unsigned int>(keylen)};
SECItem NullSecItem = {siBuffer, NULL, 0};
if(NULL == (Slot = PK11_GetInternalKeySlot())){
return false;
}
if(NULL == (pKey = PK11_ImportSymKey(Slot, CKM_SHA_1_HMAC, PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){
if(NULL == (pKey = PK11_ImportSymKey(Slot, (is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){
PK11_FreeSlot(Slot);
return false;
}
if(NULL == (Context = PK11_CreateContextBySymKey(CKM_SHA_1_HMAC, CKA_SIGN, pKey, &NullSecItem))){
if(NULL == (Context = PK11_CreateContextBySymKey((is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), CKA_SIGN, pKey, &NullSecItem))){
PK11_FreeSymKey(pKey);
PK11_FreeSlot(Slot);
return false;
@ -132,6 +132,16 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
return true;
}
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
{
return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, false);
}
bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
{
return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, true);
}
//-------------------------------------------------------------------
// Utility Function for MD5
//-------------------------------------------------------------------
@ -193,6 +203,86 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
return result;
}
//-------------------------------------------------------------------
// Utility Function for SHA256
//-------------------------------------------------------------------
size_t get_sha256_digest_length(void)
{
return SHA256_LENGTH;
}
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
{
(*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
return false;
}
PK11Context* sha256ctx;
unsigned int sha256outlen;
sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256);
PK11_DigestOp(sha256ctx, data, datalen);
PK11_DigestFinal(sha256ctx, *digest, &sha256outlen, *digestlen);
PK11_DestroyContext(sha256ctx, PR_TRUE);
*digestlen = sha256outlen;
return true;
}
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
{
PK11Context* sha256ctx;
unsigned char buf[512];
ssize_t bytes;
unsigned char* result;
unsigned int sha256outlen;
if(-1 == size){
struct stat st;
if(-1 == fstat(fd, &st)){
return NULL;
}
size = static_cast<ssize_t>(st.st_size);
}
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512);
sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256);
for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes);
if(0 == bytes){
// end of file
break;
}else if(-1 == bytes){
// error
DPRNNN("file read error(%d)", errno);
return NULL;
}
PK11_DigestOp(sha256ctx, buf, bytes);
memset(buf, 0, 512);
}
if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
PK11_DestroyContext(sha256ctx, PR_TRUE);
return NULL;
}
PK11_DigestFinal(sha256ctx, result, &sha256outlen, get_sha256_digest_length());
PK11_DestroyContext(sha256ctx, PR_TRUE);
if(-1 == lseek(fd, start, SEEK_SET)){
free(result);
return NULL;
}
return result;
}
/*
* Local variables:
* tab-width: 4

95
src/openssl_auth.cpp
View File

@ -32,6 +32,7 @@
#include <openssl/evp.h>
#include <openssl/hmac.h>
#include <openssl/md5.h>
#include <openssl/sha.h>
#include <openssl/crypto.h>
#include <openssl/err.h>
#include <string>
@ -182,7 +183,7 @@ bool s3fs_destroy_crypt_mutex(void)
//-------------------------------------------------------------------
// Utility Function for HMAC
//-------------------------------------------------------------------
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen, bool is_sha256)
{
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
return false;
@ -191,11 +192,25 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
if(NULL == ((*digest) = (unsigned char*)malloc(*digestlen))){
return false;
}
HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen);
if(is_sha256){
HMAC(EVP_sha256(), key, keylen, data, datalen, *digest, digestlen);
}else{
HMAC(EVP_sha1(), key, keylen, data, datalen, *digest, digestlen);
}
return true;
}
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
{
return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, false);
}
bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
{
return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, true);
}
//-------------------------------------------------------------------
// Utility Function for MD5
//-------------------------------------------------------------------
@ -255,6 +270,82 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
return result;
}
//-------------------------------------------------------------------
// Utility Function for SHA256
//-------------------------------------------------------------------
size_t get_sha256_digest_length(void)
{
return SHA256_DIGEST_LENGTH;
}
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
{
(*digestlen) = EVP_MAX_MD_SIZE * sizeof(unsigned char);
if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
return false;
}
const EVP_MD* md = EVP_get_digestbyname("sha256");
EVP_MD_CTX* mdctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(mdctx, md, NULL);
EVP_DigestUpdate(mdctx, data, datalen);
EVP_DigestFinal_ex(mdctx, *digest, digestlen);
EVP_MD_CTX_destroy(mdctx);
return true;
}
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
{
const EVP_MD* md = EVP_get_digestbyname("sha256");
EVP_MD_CTX* sha256ctx = EVP_MD_CTX_create();
EVP_DigestInit_ex(sha256ctx, md, NULL);
char buf[512];
ssize_t bytes;
unsigned char* result;
if(-1 == size){
struct stat st;
if(-1 == fstat(fd, &st)){
return NULL;
}
size = static_cast<ssize_t>(st.st_size);
}
// seek to top of file.
if(-1 == lseek(fd, start, SEEK_SET)){
return NULL;
}
memset(buf, 0, 512);
for(ssize_t total = 0; total < size; total += bytes){
bytes = 512 < (size - total) ? 512 : (size - total);
bytes = read(fd, buf, bytes);
if(0 == bytes){
// end of file
break;
}else if(-1 == bytes){
// error
DPRNNN("file read error(%d)", errno);
return NULL;
}
EVP_DigestUpdate(sha256ctx, buf, bytes);
memset(buf, 0, 512);
}
if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
return NULL;
}
EVP_DigestFinal_ex(sha256ctx, result, NULL);
EVP_MD_CTX_destroy(sha256ctx);
if(-1 == lseek(fd, start, SEEK_SET)){
free(result);
return NULL;
}
return result;
}
/*
* Local variables:
* tab-width: 4

124
src/s3fs.cpp
View File

@ -84,10 +84,12 @@ bool foreground = false;
bool foreground2 = false;
bool nomultipart = false;
bool pathrequeststyle = false;
bool is_specified_endpoint = false;
std::string program_name;
std::string service_path = "/";
std::string host = "http://s3.amazonaws.com";
std::string bucket = "";
std::string endpoint = "us-east-1";
//-------------------------------------------------------------------
// Static valiables
@ -2261,7 +2263,9 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
{
int result;
string s3_realpath;
string query;
string query_delimiter;;
string query_prefix;;
string query_maxkey;;
string next_marker = "";
bool truncated = true;
S3fsCurl s3fscurl;
@ -2271,31 +2275,34 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
FPRNN("[path=%s]", path);
if(delimiter && 0 < strlen(delimiter)){
query += "delimiter=";
query += delimiter;
query += "&";
query_delimiter += "delimiter=";
query_delimiter += delimiter;
query_delimiter += "&";
}
query += "prefix=";
query_prefix += "&prefix=";
s3_realpath = get_realpath(path);
if(0 == s3_realpath.length() || '/' != s3_realpath[s3_realpath.length() - 1]){
// last word must be "/"
query += urlEncode(s3_realpath.substr(1) + "/");
query_prefix += urlEncode(s3_realpath.substr(1) + "/");
}else{
query += urlEncode(s3_realpath.substr(1));
query_prefix += urlEncode(s3_realpath.substr(1));
}
if (check_content_only){
query += "&max-keys=1";
query_maxkey += "max-keys=1";
}else{
query += "&max-keys=1000";
query_maxkey += "max-keys=1000";
}
while(truncated){
string each_query = query;
string each_query = query_delimiter;
if(next_marker != ""){
each_query += "&marker=" + urlEncode(next_marker);
each_query += "marker=" + urlEncode(next_marker) + "&";
next_marker = "";
}
each_query += query_maxkey;
each_query += query_prefix;
// request
if(0 != (result = s3fscurl.ListBucketRequest(path, each_query.c_str()))){
DPRN("ListBucketRequest returns with error.");
@ -2973,6 +2980,41 @@ static int s3fs_utility_mode(void)
return result;
}
//
// If calling with wrong region, s3fs gets following error body as 400 erro code.
// "<Error><Code>AuthorizationHeaderMalformed</Code><Message>The authorization header is
// malformed; the region 'us-east-1' is wrong; expecting 'ap-northeast-1'</Message>
// <Region>ap-northeast-1</Region><RequestId>...</RequestId><HostId>...</HostId>
// </Error>"
//
// So this is cheep codes but s3fs should get correct reagion automatically.
//
static bool check_region_error(const char* pbody, string& expectregion)
{
if(!pbody){
return false;
}
const char* region;
const char* regionend;
if(NULL == (region = strcasestr(pbody, "<Message>The authorization header is malformed; the region "))){
return false;
}
if(NULL == (region = strcasestr(region, "expecting \'"))){
return false;
}
region += strlen("expecting \'");
if(NULL == (regionend = strchr(region, '\''))){
return false;
}
string strtmp(region, (regionend - region));
if(0 == strtmp.length()){
return false;
}
expectregion = strtmp;
return true;
}
static int s3fs_check_service(void)
{
FPRN("check services.");
@ -2984,12 +3026,61 @@ static int s3fs_check_service(void)
}
S3fsCurl s3fscurl;
if(0 != s3fscurl.CheckBucket()){
if(-1 == s3fscurl.CheckBucket()){
fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
return EXIT_FAILURE;
}
long responseCode = s3fscurl.GetLastResponseCode();
if(responseCode == 400){
if(!S3fsCurl::IsSignatureV4()){
// signature version 2
fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
return EXIT_FAILURE;
}
if(is_specified_endpoint){
// if specifies endpoint, do not retry to connect.
fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
return EXIT_FAILURE;
}
// check region error for signature version 4
BodyData* body = s3fscurl.GetBodyData();
string expectregion;
if(check_region_error(body->str(), expectregion)){
// not specified endpoint, so try to connect to expected region.
LOWSYSLOGPRINT(LOG_ERR, "Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str());
FPRN("Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str());
endpoint = expectregion;
// retry to check
s3fscurl.DestroyCurlHandle();
if(-1 == s3fscurl.CheckBucket()){
fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
return EXIT_FAILURE;
}
responseCode = s3fscurl.GetLastResponseCode();
}
if(responseCode == 400){
// retry to use sigv2
LOWSYSLOGPRINT(LOG_ERR, "Could not connect, so retry to connect by signature version 2.");
FPRN("Could not connect, so retry to connect by signature version 2.");
S3fsCurl::SetSignatureV4();
// retry to check
s3fscurl.DestroyCurlHandle();
if(-1 == s3fscurl.CheckBucket()){
fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
return EXIT_FAILURE;
}
responseCode = s3fscurl.GetLastResponseCode();
if(responseCode == 400){
fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
return EXIT_FAILURE;
}
}
}
if(responseCode == 403){
fprintf(stderr, "%s: invalid credentials\n", program_name.c_str());
return EXIT_FAILURE;
@ -3759,6 +3850,15 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
}
return 0;
}
if(0 == strcmp(arg, "sigv2")){
S3fsCurl::SetSignatureV4(false);
return 0;
}
if(0 == STR2NCMP(arg, "endpoint=")){
endpoint = strchr(arg, '=') + sizeof(char);
is_specified_endpoint = true;
return 0;
}
if(0 == strcmp(arg, "use_path_request_style")){
pathrequeststyle = true;
return 0;

5
src/s3fs_auth.h
View File

@ -29,6 +29,7 @@
char* s3fs_base64(unsigned char* input, size_t length);
std::string s3fs_get_content_md5(int fd);
std::string s3fs_md5sum(int fd, off_t start, ssize_t size);
std::string s3fs_sha256sum(int fd, off_t start, ssize_t size);
//
// in xxxxxx_auth.cpp
@ -39,8 +40,12 @@ bool s3fs_destroy_global_ssl(void);
bool s3fs_init_crypt_mutex(void);
bool s3fs_destroy_crypt_mutex(void);
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen);
bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen);
size_t get_md5_digest_length(void);
unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size);
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen);
size_t get_sha256_digest_length(void);
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size);
#endif // S3FS_AUTH_H_

13
src/s3fs_util.cpp
View File

@ -976,6 +976,19 @@ void show_help (void)
" url (default=\"http://s3.amazonaws.com\")\n"
" - sets the url to use to access amazon s3\n"
"\n"
" endpoint (default=\"us-east-1\")\n"
" - sets the endpoint to use on signatue version 4\n"
" If this option is not specified, s3fs uses \"us-east-1\" region as\n"
" the default. If the s3fs could not connect to the region specified\n"
" by this option, s3fs could not run. But if you do not specify this\n"
" option, and if you can not connect with the default region, s3fs\n"
" will retry to automatically connect to the other region. So s3fs\n"
" can know the correct region name, because s3fs can find it in an\n"
" error from the S3 server.\n"
"\n"
" sigv2 (default is signature version 4)\n"
" - sets signing AWS requests by sing Signature Version 2\n"
"\n"
" nomultipart (disable multipart uploads)\n"
"\n"
" enable_content_md5 (default is disable)\n"

56
src/string_util.cpp
View File

@ -125,6 +125,10 @@ string urlEncode(const string &s)
for (unsigned i = 0; i < s.length(); ++i) {
if (s[i] == '/') { // Note- special case for fuse paths...
result += s[i];
}else if (s[i] == '=') { // Note- special case for s3...
result += s[i];
}else if (s[i] == '&') { // Note- special case for s3...
result += s[i];
} else if (isalnum(s[i])) {
result += s[i];
} else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') {
@ -139,7 +143,36 @@ string urlEncode(const string &s)
result += hexAlphabet[static_cast<unsigned char>(s[i]) % 16];
}
}
return result;
}
/**
* urlEncode a fuse path,
* taking into special consideration "/",
* otherwise regular urlEncode.
*/
string urlEncode2(const string &s)
{
string result;
for (unsigned i = 0; i < s.length(); ++i) {
if (s[i] == '=') { // Note- special case for fuse paths...
result += s[i];
}else if (s[i] == '&') { // Note- special case for s3...
result += s[i];
} else if (isalnum(s[i])) {
result += s[i];
} else if (s[i] == '.' || s[i] == '-' || s[i] == '*' || s[i] == '_') {
result += s[i];
} else if (s[i] == ' ') {
result += '%';
result += '2';
result += '0';
} else {
result += "%";
result += hexAlphabet[static_cast<unsigned char>(s[i]) / 16];
result += hexAlphabet[static_cast<unsigned char>(s[i]) % 16];
}
}
return result;
}
@ -173,7 +206,7 @@ bool get_keyword_value(string& target, const char* keyword, string& value)
* Returns the current date
* in a format suitable for a HTTP request header.
*/
string get_date()
string get_date_rfc850()
{
char buf[100];
time_t t = time(NULL);
@ -181,6 +214,27 @@ string get_date()
return buf;
}
void get_date_sigv3(string& date, string& date8601)
{
time_t tm = time(NULL);
date = get_date_string(tm);
date8601 = get_date_iso8601(tm);
}
string get_date_string(time_t tm)
{
char buf[100];
strftime(buf, sizeof(buf), "%Y%m%d", gmtime(&tm));
return buf;
}
string get_date_iso8601(time_t tm)
{
char buf[100];
strftime(buf, sizeof(buf), "%Y%m%dT%H%M%SZ", gmtime(&tm));
return buf;
}
/*
* Local variables:
* tab-width: 4

6
src/string_util.h
View File

@ -46,8 +46,12 @@ std::string trim_right(const std::string &s, const std::string &t = SPACES);
std::string trim(const std::string &s, const std::string &t = SPACES);
std::string lower(std::string s);
std::string IntToStr(int);
std::string get_date();
std::string get_date_rfc850(void);
void get_date_sigv3(std::string& date, std::string& date8601);
std::string get_date_string(time_t tm);
std::string get_date_iso8601(time_t tm);
std::string urlEncode(const std::string &s);
std::string urlEncode2(const std::string &s);
bool get_keyword_value(std::string& target, const char* keyword, std::string& value);
#endif // S3FS_STRING_UTIL_H_