mirror of
https://github.com/s3fs-fuse/s3fs-fuse.git
synced 2024-11-05 04:17:52 +00:00
Supported signature version 4 for GnuTLS/NSS and automatically set endpoint/sigv2
This commit is contained in:
Takeshi Nakatani
parent
4f953f9bd7
commit
1424f87754
@ -151,6 +151,10 @@ sets the url to use to access Amazon S3. If you want to use HTTPS, then you can
|
||||
.TP
|
||||
\fB\-o\fR endpoint (default="us-east-1")
|
||||
sets the endpoint to use.
|
||||
If this option is not specified, s3fs uses \"us-east-1\" region as the default.
|
||||
If the s3fs could not connect to the region specified by this option, s3fs could not run.
|
||||
But if you do not specify this option, and if you can not connect with the default region, s3fs will retry to automatically connect to the other region.
|
||||
So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server.
|
||||
.TP
|
||||
\fB\-o\fR sigv2 (default is signature version 4)
|
||||
sets signing AWS requests by sing Signature Version 2.
|
||||
|
||||
@ -2577,9 +2577,6 @@ int S3fsCurl::CheckBucket(void)
|
||||
if (result != 0) {
|
||||
DPRN("Check bucket failed, S3 response: %s", (bodydata ? bodydata->str() : ""));
|
||||
}
|
||||
delete bodydata;
|
||||
bodydata = NULL;
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
@ -120,6 +120,25 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
|
||||
return false;
|
||||
}
|
||||
|
||||
if(NULL == (*digest = (unsigned char*)malloc(SHA256_DIGEST_SIZE))){
|
||||
return false;
|
||||
}
|
||||
|
||||
struct hmac_sha256_ctx ctx_hmac;
|
||||
hmac_sha256_set_key(&ctx_hmac, keylen, reinterpret_cast<const uint8_t*>(key));
|
||||
hmac_sha256_update(&ctx_hmac, datalen, reinterpret_cast<const uint8_t*>(data));
|
||||
hmac_sha256_digest(&ctx_hmac, SHA256_DIGEST_SIZE, reinterpret_cast<uint8_t*>(*digest));
|
||||
*digestlen = SHA256_DIGEST_SIZE;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
#else // USE_GNUTLS_NETTLE
|
||||
|
||||
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
@ -142,6 +161,26 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
|
||||
return false;
|
||||
}
|
||||
|
||||
if(0 >= (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA256))){
|
||||
return false;
|
||||
}
|
||||
if(NULL == (*digest = (unsigned char*)malloc(*digestlen + 1))){
|
||||
return false;
|
||||
}
|
||||
if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA256, key, keylen, data, datalen, *digest)){
|
||||
free(*digest);
|
||||
*digest = NULL;
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
#endif // USE_GNUTLS_NETTLE
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
@ -256,6 +295,154 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
|
||||
|
||||
#endif // USE_GNUTLS_NETTLE
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for SHA256
|
||||
//-------------------------------------------------------------------
|
||||
#define SHA256_DIGEST_LENGTH 32
|
||||
|
||||
size_t get_sha256_digest_length(void)
|
||||
{
|
||||
return SHA256_DIGEST_LENGTH;
|
||||
}
|
||||
|
||||
#ifdef USE_GNUTLS_NETTLE
|
||||
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
(*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
|
||||
if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
|
||||
return false;
|
||||
}
|
||||
|
||||
struct sha256_ctx ctx_sha256;
|
||||
sha256_init(&ctx_sha256);
|
||||
sha256_update(&ctx_sha256, datalen, data);
|
||||
sha256_digest(&ctx_sha256, *digestlen, *digest);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
struct sha256_ctx ctx_sha256;
|
||||
unsigned char buf[512];
|
||||
ssize_t bytes;
|
||||
unsigned char* result;
|
||||
|
||||
// seek to top of file.
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(buf, 0, 512);
|
||||
sha256_init(&ctx_sha256);
|
||||
|
||||
for(ssize_t total = 0; total < size; total += bytes){
|
||||
bytes = 512 < (size - total) ? 512 : (size - total);
|
||||
bytes = read(fd, buf, bytes);
|
||||
if(0 == bytes){
|
||||
// end of file
|
||||
break;
|
||||
}else if(-1 == bytes){
|
||||
// error
|
||||
DPRNNN("file read error(%d)", errno);
|
||||
return NULL;
|
||||
}
|
||||
sha256_update(&ctx_sha256, bytes, buf);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
|
||||
return NULL;
|
||||
}
|
||||
sha256_digest(&ctx_sha256, get_sha256_digest_length(), result);
|
||||
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
#else // USE_GNUTLS_NETTLE
|
||||
|
||||
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
(*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
|
||||
if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
|
||||
return false;
|
||||
}
|
||||
|
||||
gcry_md_hd_t ctx_sha256;
|
||||
gcry_error_t err;
|
||||
if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){
|
||||
DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
|
||||
free(*digest);
|
||||
return false;
|
||||
}
|
||||
gcry_md_write(ctx_sha256, data, datalen);
|
||||
memcpy(*digest, gcry_md_read(ctx_sha256, 0), *digestlen);
|
||||
gcry_md_close(ctx_sha256);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
gcry_md_hd_t ctx_sha256;
|
||||
gcry_error_t err;
|
||||
char buf[512];
|
||||
ssize_t bytes;
|
||||
unsigned char* result;
|
||||
|
||||
if(-1 == size){
|
||||
struct stat st;
|
||||
if(-1 == fstat(fd, &st)){
|
||||
return NULL;
|
||||
}
|
||||
size = static_cast<ssize_t>(st.st_size);
|
||||
}
|
||||
|
||||
// seek to top of file.
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(buf, 0, 512);
|
||||
if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){
|
||||
DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err));
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for(ssize_t total = 0; total < size; total += bytes){
|
||||
bytes = 512 < (size - total) ? 512 : (size - total);
|
||||
bytes = read(fd, buf, bytes);
|
||||
if(0 == bytes){
|
||||
// end of file
|
||||
break;
|
||||
}else if(-1 == bytes){
|
||||
// error
|
||||
DPRNNN("file read error(%d)", errno);
|
||||
return NULL;
|
||||
}
|
||||
gcry_md_write(ctx_sha256, buf, bytes);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
|
||||
return NULL;
|
||||
}
|
||||
memcpy(result, gcry_md_read(ctx_sha256, 0), get_sha256_digest_length());
|
||||
gcry_md_close(ctx_sha256);
|
||||
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
#endif // USE_GNUTLS_NETTLE
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
|
||||
@ -83,7 +83,7 @@ bool s3fs_destroy_crypt_mutex(void)
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for HMAC
|
||||
//-------------------------------------------------------------------
|
||||
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen, bool is_sha256)
|
||||
{
|
||||
if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){
|
||||
return false;
|
||||
@ -94,17 +94,17 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
|
||||
PK11Context* Context;
|
||||
SECStatus SecStatus;
|
||||
unsigned char tmpdigest[64];
|
||||
SECItem KeySecItem = {siBuffer, reinterpret_cast<unsigned char*>(const_cast<void*>(key)), keylen};
|
||||
SECItem KeySecItem = {siBuffer, reinterpret_cast<unsigned char*>(const_cast<void*>(key)), static_cast<unsigned int>(keylen)};
|
||||
SECItem NullSecItem = {siBuffer, NULL, 0};
|
||||
|
||||
if(NULL == (Slot = PK11_GetInternalKeySlot())){
|
||||
return false;
|
||||
}
|
||||
if(NULL == (pKey = PK11_ImportSymKey(Slot, CKM_SHA_1_HMAC, PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){
|
||||
if(NULL == (pKey = PK11_ImportSymKey(Slot, (is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){
|
||||
PK11_FreeSlot(Slot);
|
||||
return false;
|
||||
}
|
||||
if(NULL == (Context = PK11_CreateContextBySymKey(CKM_SHA_1_HMAC, CKA_SIGN, pKey, &NullSecItem))){
|
||||
if(NULL == (Context = PK11_CreateContextBySymKey((is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), CKA_SIGN, pKey, &NullSecItem))){
|
||||
PK11_FreeSymKey(pKey);
|
||||
PK11_FreeSlot(Slot);
|
||||
return false;
|
||||
@ -132,6 +132,16 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t
|
||||
return true;
|
||||
}
|
||||
|
||||
bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, false);
|
||||
}
|
||||
|
||||
bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, true);
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for MD5
|
||||
//-------------------------------------------------------------------
|
||||
@ -193,6 +203,86 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size)
|
||||
return result;
|
||||
}
|
||||
|
||||
//-------------------------------------------------------------------
|
||||
// Utility Function for SHA256
|
||||
//-------------------------------------------------------------------
|
||||
size_t get_sha256_digest_length(void)
|
||||
{
|
||||
return SHA256_LENGTH;
|
||||
}
|
||||
|
||||
bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen)
|
||||
{
|
||||
(*digestlen) = static_cast<unsigned int>(get_sha256_digest_length());
|
||||
if(NULL == ((*digest) = reinterpret_cast<unsigned char*>(malloc(*digestlen)))){
|
||||
return false;
|
||||
}
|
||||
|
||||
PK11Context* sha256ctx;
|
||||
unsigned int sha256outlen;
|
||||
sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256);
|
||||
|
||||
PK11_DigestOp(sha256ctx, data, datalen);
|
||||
PK11_DigestFinal(sha256ctx, *digest, &sha256outlen, *digestlen);
|
||||
PK11_DestroyContext(sha256ctx, PR_TRUE);
|
||||
*digestlen = sha256outlen;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size)
|
||||
{
|
||||
PK11Context* sha256ctx;
|
||||
unsigned char buf[512];
|
||||
ssize_t bytes;
|
||||
unsigned char* result;
|
||||
unsigned int sha256outlen;
|
||||
|
||||
if(-1 == size){
|
||||
struct stat st;
|
||||
if(-1 == fstat(fd, &st)){
|
||||
return NULL;
|
||||
}
|
||||
size = static_cast<ssize_t>(st.st_size);
|
||||
}
|
||||
|
||||
// seek to top of file.
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
return NULL;
|
||||
}
|
||||
|
||||
memset(buf, 0, 512);
|
||||
sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256);
|
||||
|
||||
for(ssize_t total = 0; total < size; total += bytes){
|
||||
bytes = 512 < (size - total) ? 512 : (size - total);
|
||||
bytes = read(fd, buf, bytes);
|
||||
if(0 == bytes){
|
||||
// end of file
|
||||
break;
|
||||
}else if(-1 == bytes){
|
||||
// error
|
||||
DPRNNN("file read error(%d)", errno);
|
||||
return NULL;
|
||||
}
|
||||
PK11_DigestOp(sha256ctx, buf, bytes);
|
||||
memset(buf, 0, 512);
|
||||
}
|
||||
if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){
|
||||
PK11_DestroyContext(sha256ctx, PR_TRUE);
|
||||
return NULL;
|
||||
}
|
||||
PK11_DigestFinal(sha256ctx, result, &sha256outlen, get_sha256_digest_length());
|
||||
PK11_DestroyContext(sha256ctx, PR_TRUE);
|
||||
|
||||
if(-1 == lseek(fd, start, SEEK_SET)){
|
||||
free(result);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
/*
|
||||
* Local variables:
|
||||
* tab-width: 4
|
||||
|
||||
90
src/s3fs.cpp
90
src/s3fs.cpp
@ -84,6 +84,7 @@ bool foreground = false;
|
||||
bool foreground2 = false;
|
||||
bool nomultipart = false;
|
||||
bool pathrequeststyle = false;
|
||||
bool is_specified_endpoint = false;
|
||||
std::string program_name;
|
||||
std::string service_path = "/";
|
||||
std::string host = "http://s3.amazonaws.com";
|
||||
@ -2979,6 +2980,41 @@ static int s3fs_utility_mode(void)
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// If calling with wrong region, s3fs gets following error body as 400 erro code.
|
||||
// "<Error><Code>AuthorizationHeaderMalformed</Code><Message>The authorization header is
|
||||
// malformed; the region 'us-east-1' is wrong; expecting 'ap-northeast-1'</Message>
|
||||
// <Region>ap-northeast-1</Region><RequestId>...</RequestId><HostId>...</HostId>
|
||||
// </Error>"
|
||||
//
|
||||
// So this is cheep codes but s3fs should get correct reagion automatically.
|
||||
//
|
||||
static bool check_region_error(const char* pbody, string& expectregion)
|
||||
{
|
||||
if(!pbody){
|
||||
return false;
|
||||
}
|
||||
const char* region;
|
||||
const char* regionend;
|
||||
if(NULL == (region = strcasestr(pbody, "<Message>The authorization header is malformed; the region "))){
|
||||
return false;
|
||||
}
|
||||
if(NULL == (region = strcasestr(region, "expecting \'"))){
|
||||
return false;
|
||||
}
|
||||
region += strlen("expecting \'");
|
||||
if(NULL == (regionend = strchr(region, '\''))){
|
||||
return false;
|
||||
}
|
||||
string strtmp(region, (regionend - region));
|
||||
if(0 == strtmp.length()){
|
||||
return false;
|
||||
}
|
||||
expectregion = strtmp;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static int s3fs_check_service(void)
|
||||
{
|
||||
FPRN("check services.");
|
||||
@ -2990,12 +3026,61 @@ static int s3fs_check_service(void)
|
||||
}
|
||||
|
||||
S3fsCurl s3fscurl;
|
||||
if(0 != s3fscurl.CheckBucket()){
|
||||
if(-1 == s3fscurl.CheckBucket()){
|
||||
fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
long responseCode = s3fscurl.GetLastResponseCode();
|
||||
|
||||
if(responseCode == 400){
|
||||
if(!S3fsCurl::IsSignatureV4()){
|
||||
// signature version 2
|
||||
fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
if(is_specified_endpoint){
|
||||
// if specifies endpoint, do not retry to connect.
|
||||
fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
|
||||
// check region error for signature version 4
|
||||
BodyData* body = s3fscurl.GetBodyData();
|
||||
string expectregion;
|
||||
if(check_region_error(body->str(), expectregion)){
|
||||
// not specified endpoint, so try to connect to expected region.
|
||||
LOWSYSLOGPRINT(LOG_ERR, "Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str());
|
||||
FPRN("Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str());
|
||||
endpoint = expectregion;
|
||||
|
||||
// retry to check
|
||||
s3fscurl.DestroyCurlHandle();
|
||||
if(-1 == s3fscurl.CheckBucket()){
|
||||
fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
responseCode = s3fscurl.GetLastResponseCode();
|
||||
}
|
||||
|
||||
if(responseCode == 400){
|
||||
// retry to use sigv2
|
||||
LOWSYSLOGPRINT(LOG_ERR, "Could not connect, so retry to connect by signature version 2.");
|
||||
FPRN("Could not connect, so retry to connect by signature version 2.");
|
||||
S3fsCurl::SetSignatureV4();
|
||||
|
||||
// retry to check
|
||||
s3fscurl.DestroyCurlHandle();
|
||||
if(-1 == s3fscurl.CheckBucket()){
|
||||
fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
responseCode = s3fscurl.GetLastResponseCode();
|
||||
if(responseCode == 400){
|
||||
fprintf(stderr, "%s: Bad Request\n", program_name.c_str());
|
||||
return EXIT_FAILURE;
|
||||
}
|
||||
}
|
||||
}
|
||||
if(responseCode == 403){
|
||||
fprintf(stderr, "%s: invalid credentials\n", program_name.c_str());
|
||||
return EXIT_FAILURE;
|
||||
@ -3770,7 +3855,8 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
|
||||
return 0;
|
||||
}
|
||||
if(0 == STR2NCMP(arg, "endpoint=")){
|
||||
endpoint = strchr(arg, '=') + sizeof(char);
|
||||
endpoint = strchr(arg, '=') + sizeof(char);
|
||||
is_specified_endpoint = true;
|
||||
return 0;
|
||||
}
|
||||
if(0 == strcmp(arg, "use_path_request_style")){
|
||||
|
||||
@ -977,7 +977,14 @@ void show_help (void)
|
||||
" - sets the url to use to access amazon s3\n"
|
||||
"\n"
|
||||
" endpoint (default=\"us-east-1\")\n"
|
||||
" - sets the endpoint to use\n"
|
||||
" - sets the endpoint to use on signatue version 4\n"
|
||||
" If this option is not specified, s3fs uses \"us-east-1\" region as\n"
|
||||
" the default. If the s3fs could not connect to the region specified\n"
|
||||
" by this option, s3fs could not run. But if you do not specify this\n"
|
||||
" option, and if you can not connect with the default region, s3fs\n"
|
||||
" will retry to automatically connect to the other region. So s3fs\n"
|
||||
" can know the correct region name, because s3fs can find it in an\n"
|
||||
" error from the S3 server.\n"
|
||||
"\n"
|
||||
" sigv2 (default is signature version 4)\n"
|
||||
" - sets signing AWS requests by sing Signature Version 2\n"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user