octoleo
/
s3fs-fuse
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow optional issuing of ListObjectsV2 (#1583) 

This allows use of s3fs on object stores that do not implement
the V1 API.  Fixes #1573.
This commit is contained in:
Andrew Gaul 2021-02-23 09:45:13 +09:00 committed by GitHub
parent 5b5bc3114a
commit 032fcf2a47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 48 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
doc/man/s3fs.1
View File

@ -305,6 +305,10 @@ If this option is specified with nocopyapi, then s3fs ignores it.
\fB\-o\fR use_path_request_style (use legacy API calling style)
Enable compatibility with S3-like APIs which do not support the virtual-host request style, by using the older path request style.
.TP
\fB\-o\fR listobjectsv2 (use ListObjectsV2)
Issue ListObjectsV2 instead of ListObjects, useful on object
stores without ListObjects support.
.TP
\fB\-o\fR noua (suppress User-Agent header)
Usually s3fs outputs of the User-Agent in "s3fs/<version> (commit hash <hash>; <using ssl library name>)" format.
If this option is specified, s3fs suppresses the output of the User-Agent.

9
src/curl.cpp
View File

@ -136,6 +136,7 @@ off_t S3fsCurl::multipart_size = MULTIPART_SIZE; // default
off_t S3fsCurl::multipart_copy_size = 512 * 1024 * 1024; // default
signature_type_t S3fsCurl::signature_type = V2_OR_V4; // default
bool S3fsCurl::is_ua = true; // default
bool S3fsCurl::listobjectsv2 = false; // default
bool S3fsCurl::is_use_session_token= false; // default
bool S3fsCurl::requester_pays = false; // default
@ -2666,7 +2667,7 @@ void S3fsCurl::insertV2Headers()
std::string turl;
std::string server_path = type == REQTYPE_LISTBUCKET ? "/" : path;
MakeUrlResource(server_path.c_str(), resource, turl);
if(!query_string.empty() && type != REQTYPE_LISTBUCKET){
if(!query_string.empty() && type != REQTYPE_CHKBUCKET && type != REQTYPE_LISTBUCKET){
resource += "?" + query_string;
}
@ -3365,10 +3366,16 @@ int S3fsCurl::CheckBucket()
if(!CreateCurlHandle()){
return -EIO;
}
std::string urlargs;
if(S3fsCurl::IsListObjectsV2()){
query_string = "list-type=2";
urlargs = "?" + query_string;
}
std::string resource;
std::string turl;
MakeUrlResource(get_realpath("/").c_str(), resource, turl);
turl += urlargs;
url = prepare_url(turl.c_str());
path = get_realpath("/");
requestHeaders = NULL;

3
src/curl.h
View File

@ -164,6 +164,7 @@ class S3fsCurl
static off_t multipart_copy_size;
static signature_type_t signature_type;
static bool is_ua; // User-Agent
static bool listobjectsv2;
static bool requester_pays;
// variables
@ -359,6 +360,8 @@ class S3fsCurl
static bool SetUserAgentFlag(bool isset) { bool bresult = S3fsCurl::is_ua; S3fsCurl::is_ua = isset; return bresult; }
static bool IsUserAgentFlag() { return S3fsCurl::is_ua; }
static void InitUserAgent();
static bool SetListObjectsV2(bool isset) { bool bresult = S3fsCurl::listobjectsv2; S3fsCurl::listobjectsv2 = isset; return bresult; }
static bool IsListObjectsV2() { return S3fsCurl::listobjectsv2; }
static bool SetRequesterPays(bool flag) { bool old_flag = S3fsCurl::requester_pays; S3fsCurl::requester_pays = flag; return old_flag; }
static bool IsRequesterPays() { return S3fsCurl::requester_pays; }
static bool SetIMDSVersion(int version);

27
src/s3fs.cpp
View File

@ -2642,6 +2642,7 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
std::string query_delimiter;
std::string query_prefix;
std::string query_maxkey;
std::string next_continuation_token;
std::string next_marker;
bool truncated = true;
S3fsCurl s3fscurl;
@ -2672,7 +2673,16 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
}
while(truncated){
std::string each_query = query_delimiter;
// append parameters to query in alphabetical order
std::string each_query = "";
if(!next_continuation_token.empty()){
each_query += "continuation-token=" + urlEncode(next_continuation_token) + "&";
next_continuation_token = "";
}
each_query += query_delimiter;
if(S3fsCurl::IsListObjectsV2()){
each_query += "list-type=2&";
}
if(!next_marker.empty()){
each_query += "marker=" + urlEncode(next_marker) + "&";
next_marker = "";
@ -2699,11 +2709,16 @@ static int list_bucket(const char* path, S3ObjList& head, const char* delimiter,
return -EIO;
}
if(true == (truncated = is_truncated(doc))){
xmlChar* tmpch = get_next_marker(doc);
if(tmpch){
xmlChar* tmpch;
if(NULL != (tmpch = get_next_contination_token(doc))){
next_continuation_token = (char*)tmpch;
xmlFree(tmpch);
}else if(NULL != (tmpch = get_next_marker(doc))){
next_marker = (char*)tmpch;
xmlFree(tmpch);
}else{
}
if(next_continuation_token.empty() && next_marker.empty()){
// If did not specify "delimiter", s3 did not return "NextMarker".
// On this case, can use last name for next marker.
//
@ -4634,6 +4649,10 @@ static int my_fuse_opt_proc(void* data, const char* arg, int key, struct fuse_ar
S3fsCurl::SetUserAgentFlag(false);
return 0;
}
if(0 == strcmp(arg, "listobjectsv2")){
S3fsCurl::SetListObjectsV2(true);
return 0;
}
if(0 == strcmp(arg, "use_xattr")){
is_use_xattr = true;
return 0;

4
src/s3fs_help.cpp
View File

@ -382,6 +382,10 @@ static const char help_string[] =
" the virtual-host request style, by using the older path request\n"
" style.\n"
"\n"
" listobjectsv2 (use ListObjectsV2)\n"
" Issue ListObjectsV2 instead of ListObjects, useful on object\n"
" stores without ListObjects support.\n"
"\n"
" noua (suppress User-Agent header)\n"
" Usually s3fs outputs of the User-Agent in \"s3fs/<version> (commit\n"
" hash <hash>; <using ssl library name>)\" format.\n"

5
src/s3fs_xml.cpp
View File

@ -109,6 +109,11 @@ static xmlChar* get_prefix(xmlDocPtr doc)
return get_base_exp(doc, "Prefix");
}
xmlChar* get_next_contination_token(xmlDocPtr doc)
{
return get_base_exp(doc, "NextContinuationToken");
}
xmlChar* get_next_marker(xmlDocPtr doc)
{
return get_base_exp(doc, "NextMarker");

1
src/s3fs_xml.h
View File

@ -36,6 +36,7 @@
bool is_truncated(xmlDocPtr doc);
int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head);
int append_objects_from_xml(const char* path, xmlDocPtr doc, S3ObjList& head);
xmlChar* get_next_contination_token(xmlDocPtr doc);
xmlChar* get_next_marker(xmlDocPtr doc);
bool get_incomp_mpu_list(xmlDocPtr doc, incomp_mpu_list_t& list);