octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-06-04 18:10:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

multipart upload id is converted by url encode

Browse Source
This commit is contained in:
Takeshi Nakatani 2023-01-22 05:41:16 +00:00
parent dbddd762a9
commit bd8838f304
6 changed files with 77 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
src/curl.cpp
View File

@ -1219,6 +1219,7 @@ S3fsCurl* S3fsCurl::UploadMultipartPostRetryCallback(S3fsCurl* s3fscurl)
if(!get_keyword_value(s3fscurl->url, "uploadId", upload_id)){
return NULL;
}
upload_id = urlDecode(upload_id); // decode
if(!get_keyword_value(s3fscurl->url, "partNumber", part_num_str)){
return NULL;
}
@ -1266,6 +1267,7 @@ S3fsCurl* S3fsCurl::CopyMultipartPostRetryCallback(S3fsCurl* s3fscurl)
if(!get_keyword_value(s3fscurl->url, "uploadId", upload_id)){
return NULL;
}
upload_id = urlDecode(upload_id); // decode
if(!get_keyword_value(s3fscurl->url, "partNumber", part_num_str)){
return NULL;
}
@ -2789,7 +2791,7 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string
requestHeaders = curl_slist_sort_insert(requestHeaders, "x-amz-security-token", access_token.c_str());
}
uriencode = urlEncode(canonical_uri);
uriencode = urlEncodePath(canonical_uri);
StringCQ = method + "\n";
if(method == "HEAD" || method == "PUT" || method == "DELETE"){
StringCQ += uriencode + "\n";
@ -2798,11 +2800,11 @@ std::string S3fsCurl::CalcSignature(const std::string& method, const std::string
}else if(method == "GET" && is_prefix(uriencode.c_str(), "/")){
StringCQ += uriencode +"\n";
}else if(method == "GET" && !is_prefix(uriencode.c_str(), "/")){
StringCQ += "/\n" + urlEncode2(canonical_uri) +"\n";
StringCQ += "/\n" + urlEncodeQuery(canonical_uri) +"\n";
}else if(method == "POST"){
StringCQ += uriencode + "\n";
}
StringCQ += urlEncode2(query_string) + "\n";
StringCQ += urlEncodeQuery(query_string) + "\n";
StringCQ += get_canonical_headers(requestHeaders) + "\n";
StringCQ += get_sorted_header_keys(requestHeaders) + "\n";
StringCQ += payload_hash;
@ -3921,7 +3923,11 @@ int S3fsCurl::CompleteMultipartPostRequest(const char* tpath, const std::string&
std::string turl;
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
query_string = "uploadId=" + upload_id;
// [NOTE]
// Encode the upload_id here.
// In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included.
//
query_string = "uploadId=" + urlEncodeGeneral(upload_id);
turl += "?" + query_string;
url = prepare_url(turl.c_str());
path = get_realpath(tpath);
@ -4039,7 +4045,11 @@ int S3fsCurl::AbortMultipartUpload(const char* tpath, const std::string& upload_
std::string turl;
MakeUrlResource(get_realpath(tpath).c_str(), resource, turl);
query_string = "uploadId=" + upload_id;
// [NOTE]
// Encode the upload_id here.
// In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included.
//
query_string = "uploadId=" + urlEncodeGeneral(upload_id);
turl += "?" + query_string;
url = prepare_url(turl.c_str());
path = get_realpath(tpath);
@ -4101,7 +4111,12 @@ int S3fsCurl::UploadMultipartPostSetup(const char* tpath, int part_num, const st
}
// make request
query_string = "partNumber=" + str(part_num) + "&uploadId=" + upload_id;
//
// [NOTE]
// Encode the upload_id here.
// In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included.
//
query_string = "partNumber=" + str(part_num) + "&uploadId=" + urlEncodeGeneral(upload_id);
std::string urlargs = "?" + query_string;
std::string resource;
std::string turl;
@ -4169,7 +4184,11 @@ int S3fsCurl::CopyMultipartPostSetup(const char* from, const char* to, int part_
if(!from || !to){
return -EINVAL;
}
query_string = "partNumber=" + str(part_num) + "&uploadId=" + upload_id;
// [NOTE]
// Encode the upload_id here.
// In compatible S3 servers(Cloudflare, etc), there are cases where characters that require URL encoding are included.
//
query_string = "partNumber=" + str(part_num) + "&uploadId=" + urlEncodeGeneral(upload_id);
std::string urlargs = "?" + query_string;
std::string resource;
std::string turl;

2
src/curl_util.cpp
View File

@ -247,7 +247,7 @@ bool MakeUrlResource(const char* realpath, std::string& resourcepath, std::strin
if(!realpath){
return false;
}
resourcepath = urlEncode(service_path + S3fsCred::GetBucket() + realpath);
resourcepath = urlEncodePath(service_path + S3fsCred::GetBucket() + realpath);
url = s3host + resourcepath;
return true;
}

2
src/fdcache_entity.cpp
View File

@ -2410,7 +2410,7 @@ int FdEntity::UploadPending(int fd, AutoLock::Type type)
}else if(UPDATE_META_PENDING == pending_status){
headers_t updatemeta = orgmeta;
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(path.c_str()));
updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(path.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// put headers, no need to update mtime to avoid dead lock

26
src/s3fs.cpp
View File

@ -1364,7 +1364,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
if(update_ctime){
meta["x-amz-meta-ctime"] = s3fs_str_realtime();
}
meta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
meta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
meta["Content-Type"] = S3fsCurl::LookupMimeType(std::string(to));
meta["x-amz-metadata-directive"] = "REPLACE";
@ -1845,7 +1845,7 @@ static int s3fs_chmod(const char* _path, mode_t mode)
headers_t updatemeta;
updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime();
updatemeta["x-amz-meta-mode"] = str(mode);
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// check opened file handle.
@ -2050,7 +2050,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime();
updatemeta["x-amz-meta-uid"] = str(uid);
updatemeta["x-amz-meta-gid"] = str(gid);
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// check opened file handle.
@ -2285,7 +2285,7 @@ static int update_mctime_parent_directory(const char* _path)
updatemeta["x-amz-meta-mtime"] = str(mctime);
updatemeta["x-amz-meta-ctime"] = str(mctime);
updatemeta["x-amz-meta-atime"] = str(atime);
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
merge_headers(meta, updatemeta, true);
@ -2378,7 +2378,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2])
updatemeta["x-amz-meta-mtime"] = str(mtime);
updatemeta["x-amz-meta-ctime"] = str(ctime);
updatemeta["x-amz-meta-atime"] = str(atime);
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// check opened file handle.
@ -3268,9 +3268,9 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
s3_realpath = get_realpath(path);
if(s3_realpath.empty() || '/' != *s3_realpath.rbegin()){
// last word must be "/"
query_prefix += urlEncode(s3_realpath.substr(1) + "/");
query_prefix += urlEncodePath(s3_realpath.substr(1) + "/");
}else{
query_prefix += urlEncode(s3_realpath.substr(1));
query_prefix += urlEncodePath(s3_realpath.substr(1));
}
if (check_content_only){
// Just need to know if there are child objects in dir
@ -3284,7 +3284,7 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
// append parameters to query in alphabetical order
std::string each_query;
if(!next_continuation_token.empty()){
each_query += "continuation-token=" + urlEncode(next_continuation_token) + "&";
each_query += "continuation-token=" + urlEncodePath(next_continuation_token) + "&";
next_continuation_token = "";
}
each_query += query_delimiter;
@ -3292,7 +3292,7 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
each_query += "list-type=2&";
}
if(!next_marker.empty()){
each_query += "marker=" + urlEncode(next_marker) + "&";
each_query += "marker=" + urlEncodePath(next_marker) + "&";
next_marker = "";
}
each_query += query_maxkey;
@ -3488,7 +3488,7 @@ static bool build_inherited_xattr_value(const char* path, std::string& xattrvalu
raw_xattr_value += parent_default_value;
raw_xattr_value += "\"}";
xattrvalue = urlEncode(raw_xattr_value);
xattrvalue = urlEncodePath(raw_xattr_value);
return true;
}
@ -3596,7 +3596,7 @@ static std::string build_xattrs(const xattrs_t& xattrs)
if(strxattrs.empty()){
strxattrs = "{}";
}
strxattrs = urlEncode(strxattrs);
strxattrs = urlEncodePath(strxattrs);
return strxattrs;
}
@ -3733,7 +3733,7 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value,
std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath;
headers_t updatemeta;
updatemeta["x-amz-meta-ctime"] = s3fs_str_realtime();
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
// check opened file handle.
@ -4028,7 +4028,7 @@ static int s3fs_removexattr(const char* path, const char* name)
// set xattr all object
std::string strSourcePath = (mount_prefix.empty() && "/" == strpath) ? "//" : strpath;
headers_t updatemeta;
updatemeta["x-amz-copy-source"] = urlEncode(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-copy-source"] = urlEncodePath(service_path + S3fsCred::GetBucket() + get_realpath(strSourcePath.c_str()));
updatemeta["x-amz-metadata-directive"] = "REPLACE";
if(!xattrs.empty()){
updatemeta["x-amz-meta-xattr"] = build_xattrs(xattrs);

72
src/string_util.cpp
View File

@ -147,23 +147,32 @@ std::string trim(const std::string &s, const char *t /* = SPACES */)
}
//
// urlEncode a fuse path,
// taking into special consideration "/",
// otherwise regular urlEncode.
// Three url encode functions
//
std::string urlEncode(const std::string &s)
// urlEncodeGeneral: A general URL encoding function.
// urlEncodePath : A function that URL encodes by excluding the path
// separator('/').
// urlEncodeQuery : A function that does URL encoding by excluding
// some characters('=', '&' and '%').
// This function can be used when the target string
// contains already URL encoded strings. It also
// excludes the character () used in query strings.
// Therefore, it is a function to use as URL encoding
// for use in query strings.
//
static const char* encode_general_except_chars = ".-_~"; // For general URL encode
static const char* encode_path_except_chars = ".-_~/"; // For fuse(included path) URL encode
static const char* encode_query_except_chars = ".-_~=&%"; // For query params(and encoded string)
static std::string rawUrlEncode(const std::string &s, const char* except_chars)
{
std::string result;
for (size_t i = 0; i < s.length(); ++i) {
unsigned char c = s[i];
if (c == '/' // Note- special case for fuse paths...
|| c == '.'
|| c == '-'
|| c == '_'
|| c == '~'
|| (c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9'))
if((except_chars && NULL != strchr(except_chars, c)) ||
(c >= 'a' && c <= 'z') ||
(c >= 'A' && c <= 'Z') ||
(c >= '0' && c <= '9') )
{
result += c;
}else{
@ -174,34 +183,19 @@ std::string urlEncode(const std::string &s)
return result;
}
//
// urlEncode a fuse path,
// taking into special consideration "/",
// otherwise regular urlEncode.
//
std::string urlEncode2(const std::string &s)
std::string urlEncodeGeneral(const std::string &s)
{
std::string result;
for (size_t i = 0; i < s.length(); ++i) {
unsigned char c = s[i];
if (c == '=' // Note- special case for fuse paths...
|| c == '&' // Note- special case for s3...
|| c == '%'
|| c == '.'
|| c == '-'
|| c == '_'
|| c == '~'
|| (c >= 'a' && c <= 'z')
|| (c >= 'A' && c <= 'Z')
|| (c >= '0' && c <= '9'))
{
result += c;
}else{
result += "%";
result += s3fs_hex_upper(&c, 1);
}
}
return result;
return rawUrlEncode(s, encode_general_except_chars);
}
std::string urlEncodePath(const std::string &s)
{
return rawUrlEncode(s, encode_path_except_chars);
}
std::string urlEncodeQuery(const std::string &s)
{
return rawUrlEncode(s, encode_query_except_chars);
}
std::string urlDecode(const std::string& s)

5
src/string_util.h
View File

@ -94,8 +94,9 @@ bool convert_unixtime_from_option_arg(const char* argv, time_t& unixtime);
//
// For encoding
//
std::string urlEncode(const std::string &s);
std::string urlEncode2(const std::string &s);
std::string urlEncodeGeneral(const std::string &s);
std::string urlEncodePath(const std::string &s);
std::string urlEncodeQuery(const std::string &s);
std::string urlDecode(const std::string& s);
bool takeout_str_dquart(std::string& str);