diff --git a/doc/man/s3fs.1 b/doc/man/s3fs.1 index 438bf0a..c0f11aa 100644 --- a/doc/man/s3fs.1 +++ b/doc/man/s3fs.1 @@ -151,6 +151,10 @@ sets the url to use to access Amazon S3. If you want to use HTTPS, then you can .TP \fB\-o\fR endpoint (default="us-east-1") sets the endpoint to use. +If this option is not specified, s3fs uses \"us-east-1\" region as the default. +If the s3fs could not connect to the region specified by this option, s3fs could not run. +But if you do not specify this option, and if you can not connect with the default region, s3fs will retry to automatically connect to the other region. +So s3fs can know the correct region name, because s3fs can find it in an error from the S3 server. .TP \fB\-o\fR sigv2 (default is signature version 4) sets signing AWS requests by sing Signature Version 2. diff --git a/src/curl.cpp b/src/curl.cpp index 2fcec94..7fa85ee 100644 --- a/src/curl.cpp +++ b/src/curl.cpp @@ -2577,9 +2577,6 @@ int S3fsCurl::CheckBucket(void) if (result != 0) { DPRN("Check bucket failed, S3 response: %s", (bodydata ? bodydata->str() : "")); } - delete bodydata; - bodydata = NULL; - return result; } diff --git a/src/gnutls_auth.cpp b/src/gnutls_auth.cpp index 72de2df..9ffe958 100644 --- a/src/gnutls_auth.cpp +++ b/src/gnutls_auth.cpp @@ -120,6 +120,25 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t return true; } +bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){ + return false; + } + + if(NULL == (*digest = (unsigned char*)malloc(SHA256_DIGEST_SIZE))){ + return false; + } + + struct hmac_sha256_ctx ctx_hmac; + hmac_sha256_set_key(&ctx_hmac, keylen, reinterpret_cast(key)); + hmac_sha256_update(&ctx_hmac, datalen, reinterpret_cast(data)); + hmac_sha256_digest(&ctx_hmac, SHA256_DIGEST_SIZE, reinterpret_cast(*digest)); + *digestlen = SHA256_DIGEST_SIZE; + + return true; +} + #else // USE_GNUTLS_NETTLE bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) @@ -142,6 +161,26 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t return true; } +bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){ + return false; + } + + if(0 >= (*digestlen = gnutls_hmac_get_len(GNUTLS_MAC_SHA256))){ + return false; + } + if(NULL == (*digest = (unsigned char*)malloc(*digestlen + 1))){ + return false; + } + if(0 > gnutls_hmac_fast(GNUTLS_MAC_SHA256, key, keylen, data, datalen, *digest)){ + free(*digest); + *digest = NULL; + return false; + } + return true; +} + #endif // USE_GNUTLS_NETTLE //------------------------------------------------------------------- @@ -256,6 +295,154 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size) #endif // USE_GNUTLS_NETTLE +//------------------------------------------------------------------- +// Utility Function for SHA256 +//------------------------------------------------------------------- +#define SHA256_DIGEST_LENGTH 32 + +size_t get_sha256_digest_length(void) +{ + return SHA256_DIGEST_LENGTH; +} + +#ifdef USE_GNUTLS_NETTLE +bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen) +{ + (*digestlen) = static_cast(get_sha256_digest_length()); + if(NULL == ((*digest) = reinterpret_cast(malloc(*digestlen)))){ + return false; + } + + struct sha256_ctx ctx_sha256; + sha256_init(&ctx_sha256); + sha256_update(&ctx_sha256, datalen, data); + sha256_digest(&ctx_sha256, *digestlen, *digest); + + return true; +} + +unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) +{ + struct sha256_ctx ctx_sha256; + unsigned char buf[512]; + ssize_t bytes; + unsigned char* result; + + // seek to top of file. + if(-1 == lseek(fd, start, SEEK_SET)){ + return NULL; + } + + memset(buf, 0, 512); + sha256_init(&ctx_sha256); + + for(ssize_t total = 0; total < size; total += bytes){ + bytes = 512 < (size - total) ? 512 : (size - total); + bytes = read(fd, buf, bytes); + if(0 == bytes){ + // end of file + break; + }else if(-1 == bytes){ + // error + DPRNNN("file read error(%d)", errno); + return NULL; + } + sha256_update(&ctx_sha256, bytes, buf); + memset(buf, 0, 512); + } + if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){ + return NULL; + } + sha256_digest(&ctx_sha256, get_sha256_digest_length(), result); + + if(-1 == lseek(fd, start, SEEK_SET)){ + free(result); + return NULL; + } + + return result; +} + +#else // USE_GNUTLS_NETTLE + +bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen) +{ + (*digestlen) = static_cast(get_sha256_digest_length()); + if(NULL == ((*digest) = reinterpret_cast(malloc(*digestlen)))){ + return false; + } + + gcry_md_hd_t ctx_sha256; + gcry_error_t err; + if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){ + DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); + free(*digest); + return false; + } + gcry_md_write(ctx_sha256, data, datalen); + memcpy(*digest, gcry_md_read(ctx_sha256, 0), *digestlen); + gcry_md_close(ctx_sha256); + + return true; +} + +unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) +{ + gcry_md_hd_t ctx_sha256; + gcry_error_t err; + char buf[512]; + ssize_t bytes; + unsigned char* result; + + if(-1 == size){ + struct stat st; + if(-1 == fstat(fd, &st)){ + return NULL; + } + size = static_cast(st.st_size); + } + + // seek to top of file. + if(-1 == lseek(fd, start, SEEK_SET)){ + return NULL; + } + + memset(buf, 0, 512); + if(GPG_ERR_NO_ERROR != (err = gcry_md_open(&ctx_sha256, GCRY_MD_SHA256, 0))){ + DPRNN("SHA256 context creation failure: %s/%s", gcry_strsource(err), gcry_strerror(err)); + return NULL; + } + + for(ssize_t total = 0; total < size; total += bytes){ + bytes = 512 < (size - total) ? 512 : (size - total); + bytes = read(fd, buf, bytes); + if(0 == bytes){ + // end of file + break; + }else if(-1 == bytes){ + // error + DPRNNN("file read error(%d)", errno); + return NULL; + } + gcry_md_write(ctx_sha256, buf, bytes); + memset(buf, 0, 512); + } + if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){ + return NULL; + } + memcpy(result, gcry_md_read(ctx_sha256, 0), get_sha256_digest_length()); + gcry_md_close(ctx_sha256); + + if(-1 == lseek(fd, start, SEEK_SET)){ + free(result); + return NULL; + } + + return result; +} + +#endif // USE_GNUTLS_NETTLE + /* * Local variables: * tab-width: 4 diff --git a/src/nss_auth.cpp b/src/nss_auth.cpp index f820426..4fb11a5 100644 --- a/src/nss_auth.cpp +++ b/src/nss_auth.cpp @@ -83,7 +83,7 @@ bool s3fs_destroy_crypt_mutex(void) //------------------------------------------------------------------- // Utility Function for HMAC //------------------------------------------------------------------- -bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +static bool s3fs_HMAC_RAW(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen, bool is_sha256) { if(!key || 0 >= keylen || !data || 0 >= datalen || !digest || !digestlen){ return false; @@ -94,17 +94,17 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t PK11Context* Context; SECStatus SecStatus; unsigned char tmpdigest[64]; - SECItem KeySecItem = {siBuffer, reinterpret_cast(const_cast(key)), keylen}; + SECItem KeySecItem = {siBuffer, reinterpret_cast(const_cast(key)), static_cast(keylen)}; SECItem NullSecItem = {siBuffer, NULL, 0}; if(NULL == (Slot = PK11_GetInternalKeySlot())){ return false; } - if(NULL == (pKey = PK11_ImportSymKey(Slot, CKM_SHA_1_HMAC, PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){ + if(NULL == (pKey = PK11_ImportSymKey(Slot, (is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), PK11_OriginUnwrap, CKA_SIGN, &KeySecItem, NULL))){ PK11_FreeSlot(Slot); return false; } - if(NULL == (Context = PK11_CreateContextBySymKey(CKM_SHA_1_HMAC, CKA_SIGN, pKey, &NullSecItem))){ + if(NULL == (Context = PK11_CreateContextBySymKey((is_sha256 ? CKM_SHA256_HMAC : CKM_SHA_1_HMAC), CKA_SIGN, pKey, &NullSecItem))){ PK11_FreeSymKey(pKey); PK11_FreeSlot(Slot); return false; @@ -132,6 +132,16 @@ bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t return true; } +bool s3fs_HMAC(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, false); +} + +bool s3fs_HMAC256(const void* key, size_t keylen, const unsigned char* data, size_t datalen, unsigned char** digest, unsigned int* digestlen) +{ + return s3fs_HMAC_RAW(key, keylen, data, datalen, digest, digestlen, true); +} + //------------------------------------------------------------------- // Utility Function for MD5 //------------------------------------------------------------------- @@ -193,6 +203,86 @@ unsigned char* s3fs_md5hexsum(int fd, off_t start, ssize_t size) return result; } +//------------------------------------------------------------------- +// Utility Function for SHA256 +//------------------------------------------------------------------- +size_t get_sha256_digest_length(void) +{ + return SHA256_LENGTH; +} + +bool s3fs_sha256(const unsigned char* data, unsigned int datalen, unsigned char** digest, unsigned int* digestlen) +{ + (*digestlen) = static_cast(get_sha256_digest_length()); + if(NULL == ((*digest) = reinterpret_cast(malloc(*digestlen)))){ + return false; + } + + PK11Context* sha256ctx; + unsigned int sha256outlen; + sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256); + + PK11_DigestOp(sha256ctx, data, datalen); + PK11_DigestFinal(sha256ctx, *digest, &sha256outlen, *digestlen); + PK11_DestroyContext(sha256ctx, PR_TRUE); + *digestlen = sha256outlen; + + return true; +} + +unsigned char* s3fs_sha256hexsum(int fd, off_t start, ssize_t size) +{ + PK11Context* sha256ctx; + unsigned char buf[512]; + ssize_t bytes; + unsigned char* result; + unsigned int sha256outlen; + + if(-1 == size){ + struct stat st; + if(-1 == fstat(fd, &st)){ + return NULL; + } + size = static_cast(st.st_size); + } + + // seek to top of file. + if(-1 == lseek(fd, start, SEEK_SET)){ + return NULL; + } + + memset(buf, 0, 512); + sha256ctx = PK11_CreateDigestContext(SEC_OID_SHA256); + + for(ssize_t total = 0; total < size; total += bytes){ + bytes = 512 < (size - total) ? 512 : (size - total); + bytes = read(fd, buf, bytes); + if(0 == bytes){ + // end of file + break; + }else if(-1 == bytes){ + // error + DPRNNN("file read error(%d)", errno); + return NULL; + } + PK11_DigestOp(sha256ctx, buf, bytes); + memset(buf, 0, 512); + } + if(NULL == (result = (unsigned char*)malloc(get_sha256_digest_length()))){ + PK11_DestroyContext(sha256ctx, PR_TRUE); + return NULL; + } + PK11_DigestFinal(sha256ctx, result, &sha256outlen, get_sha256_digest_length()); + PK11_DestroyContext(sha256ctx, PR_TRUE); + + if(-1 == lseek(fd, start, SEEK_SET)){ + free(result); + return NULL; + } + + return result; +} + /* * Local variables: * tab-width: 4 diff --git a/src/s3fs.cpp b/src/s3fs.cpp index 84589e6..5bef1bb 100644 --- a/src/s3fs.cpp +++ b/src/s3fs.cpp @@ -84,6 +84,7 @@ bool foreground = false; bool foreground2 = false; bool nomultipart = false; bool pathrequeststyle = false; +bool is_specified_endpoint = false; std::string program_name; std::string service_path = "/"; std::string host = "http://s3.amazonaws.com"; @@ -2979,6 +2980,41 @@ static int s3fs_utility_mode(void) return result; } +// +// If calling with wrong region, s3fs gets following error body as 400 erro code. +// "AuthorizationHeaderMalformedThe authorization header is +// malformed; the region 'us-east-1' is wrong; expecting 'ap-northeast-1' +// ap-northeast-1...... +// " +// +// So this is cheep codes but s3fs should get correct reagion automatically. +// +static bool check_region_error(const char* pbody, string& expectregion) +{ + if(!pbody){ + return false; + } + const char* region; + const char* regionend; + if(NULL == (region = strcasestr(pbody, "The authorization header is malformed; the region "))){ + return false; + } + if(NULL == (region = strcasestr(region, "expecting \'"))){ + return false; + } + region += strlen("expecting \'"); + if(NULL == (regionend = strchr(region, '\''))){ + return false; + } + string strtmp(region, (regionend - region)); + if(0 == strtmp.length()){ + return false; + } + expectregion = strtmp; + + return true; +} + static int s3fs_check_service(void) { FPRN("check services."); @@ -2990,12 +3026,61 @@ static int s3fs_check_service(void) } S3fsCurl s3fscurl; - if(0 != s3fscurl.CheckBucket()){ + if(-1 == s3fscurl.CheckBucket()){ fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str()); return EXIT_FAILURE; } long responseCode = s3fscurl.GetLastResponseCode(); + if(responseCode == 400){ + if(!S3fsCurl::IsSignatureV4()){ + // signature version 2 + fprintf(stderr, "%s: Bad Request\n", program_name.c_str()); + return EXIT_FAILURE; + } + if(is_specified_endpoint){ + // if specifies endpoint, do not retry to connect. + fprintf(stderr, "%s: Bad Request\n", program_name.c_str()); + return EXIT_FAILURE; + } + + // check region error for signature version 4 + BodyData* body = s3fscurl.GetBodyData(); + string expectregion; + if(check_region_error(body->str(), expectregion)){ + // not specified endpoint, so try to connect to expected region. + LOWSYSLOGPRINT(LOG_ERR, "Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str()); + FPRN("Could not connect wrong region %s, so retry to connect region %s.", endpoint.c_str(), expectregion.c_str()); + endpoint = expectregion; + + // retry to check + s3fscurl.DestroyCurlHandle(); + if(-1 == s3fscurl.CheckBucket()){ + fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str()); + return EXIT_FAILURE; + } + responseCode = s3fscurl.GetLastResponseCode(); + } + + if(responseCode == 400){ + // retry to use sigv2 + LOWSYSLOGPRINT(LOG_ERR, "Could not connect, so retry to connect by signature version 2."); + FPRN("Could not connect, so retry to connect by signature version 2."); + S3fsCurl::SetSignatureV4(); + + // retry to check + s3fscurl.DestroyCurlHandle(); + if(-1 == s3fscurl.CheckBucket()){ + fprintf(stderr, "%s: Failed to access bucket.\n", program_name.c_str()); + return EXIT_FAILURE; + } + responseCode = s3fscurl.GetLastResponseCode(); + if(responseCode == 400){ + fprintf(stderr, "%s: Bad Request\n", program_name.c_str()); + return EXIT_FAILURE; + } + } + } if(responseCode == 403){ fprintf(stderr, "%s: invalid credentials\n", program_name.c_str()); return EXIT_FAILURE; @@ -3770,7 +3855,8 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar return 0; } if(0 == STR2NCMP(arg, "endpoint=")){ - endpoint = strchr(arg, '=') + sizeof(char); + endpoint = strchr(arg, '=') + sizeof(char); + is_specified_endpoint = true; return 0; } if(0 == strcmp(arg, "use_path_request_style")){ diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index f481389..484d11a 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -977,7 +977,14 @@ void show_help (void) " - sets the url to use to access amazon s3\n" "\n" " endpoint (default=\"us-east-1\")\n" - " - sets the endpoint to use\n" + " - sets the endpoint to use on signatue version 4\n" + " If this option is not specified, s3fs uses \"us-east-1\" region as\n" + " the default. If the s3fs could not connect to the region specified\n" + " by this option, s3fs could not run. But if you do not specify this\n" + " option, and if you can not connect with the default region, s3fs\n" + " will retry to automatically connect to the other region. So s3fs\n" + " can know the correct region name, because s3fs can find it in an\n" + " error from the S3 server.\n" "\n" " sigv2 (default is signature version 4)\n" " - sets signing AWS requests by sing Signature Version 2\n"