octoleo
/
s3fs-fuse
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Explicitly handle CommonPrefixes with nocompat_dir (#2212) 

Previously the test missed listing implicit directories and another
test was incorrect.  This fixes a regression from 1.91.
This commit is contained in:
Andrew Gaul 2023-07-13 21:15:34 +09:00 committed by GitHub
parent 9663215bb4
commit e650d8c55c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 84 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/s3fs.cpp
View File

@ -21,6 +21,7 @@
#include <cstdio>
#include <cstdlib>
#include <errno.h>
#include <set>
#include <unistd.h>
#include <dirent.h>
#include <sys/types.h>
@ -3067,6 +3068,7 @@ struct multi_head_callback_param
{
void* buf;
fuse_fill_dir_t filler;
std::set<std::string> *filled;
};
static bool multi_head_callback(S3fsCurl* s3fscurl, void* param)
@ -3096,6 +3098,7 @@ static bool multi_head_callback(S3fsCurl* s3fscurl, void* param)
S3FS_PRN_INFO2("Could not find %s file in stat cache.", saved_path.c_str());
pcbparam->filler(pcbparam->buf, bpath.c_str(), 0, 0);
}
pcbparam->filled->insert(bpath);
}else{
S3FS_PRN_WARN("param(multi_head_callback_param*) is NULL, then can not call filler.");
}
@ -3170,6 +3173,7 @@ static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf
S3fsMultiCurl curlmulti(S3fsCurl::GetMaxMultiRequest());
s3obj_list_t headlist;
int result = 0;
std::set<std::string> filled;
S3FS_PRN_INFO1("[path=%s][list=%zu]", path, headlist.size());
@ -3184,6 +3188,7 @@ static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf
struct multi_head_callback_param success_param;
success_param.buf = buf;
success_param.filler = filler;
success_param.filled = &filled;
curlmulti.SetSuccessCallbackParam(reinterpret_cast<void*>(&success_param));
// Not found Callback function parameter
@ -3218,6 +3223,7 @@ static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf
bpath = s3fs_wtf8_decode(bpath);
}
filler(buf, bpath.c_str(), &st, 0);
filled.insert(bpath);
continue;
}
@ -3255,6 +3261,14 @@ static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf
// Objects that could not be found by HEAD request may exist only
// as a path, so search for objects under that path.(a case of no dir object)
//
if(!support_compat_dir){
for(std::vector<std::string>::const_iterator it = head.common_prefixes.begin(); it != head.common_prefixes.end(); ++it) {
if(filled.find(*it) == filled.end()){
filler(buf, it->c_str(), 0, 0);
filled.insert(*it);
}
}
}
if(support_compat_dir && !notfound_param.notfound_list.empty()){ // [NOTE] not need to lock to access this here.
// dummy header
mode_t dirmask = umask(0); // macos does not have getumask()
@ -3291,6 +3305,7 @@ static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf
S3FS_PRN_INFO2("Could not find %s directory(no dir object) in stat cache.", dirpath.c_str());
filler(buf, base_path.c_str(), 0, 0);
}
filled.insert(base_path);
}else{
S3FS_PRN_ERR("failed adding stat cache [path=%s], but dontinue...", dirpath.c_str());
}

9
src/s3fs_xml.cpp
View File

@ -341,7 +341,7 @@ bool is_truncated(xmlDocPtr doc)
return result;
}
int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head)
int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head, bool prefix)
{
xmlXPathObjectPtr contents_xp;
xmlNodeSetPtr content_nodes;
@ -409,6 +409,9 @@ int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextP
std::string decname = get_decoded_cr_code(name);
free(name);
if(prefix){
head.common_prefixes.push_back(decname);
}
if(!head.insert(decname.c_str(), (!stretag.empty() ? stretag.c_str() : NULL), is_dir)){
S3FS_PRN_ERR("insert_object returns with error.");
xmlXPathFreeObject(key);
@ -462,8 +465,8 @@ int append_objects_from_xml(const char* path, xmlDocPtr doc, S3ObjList& head)
ex_prefix += "Prefix";
ex_etag += "ETag";
if(-1 == append_objects_from_xml_ex(prefix.c_str(), doc, ctx, ex_contents.c_str(), ex_key.c_str(), ex_etag.c_str(), 0, head) ||
-1 == append_objects_from_xml_ex(prefix.c_str(), doc, ctx, ex_cprefix.c_str(), ex_prefix.c_str(), NULL, 1, head) )
if(-1 == append_objects_from_xml_ex(prefix.c_str(), doc, ctx, ex_contents.c_str(), ex_key.c_str(), ex_etag.c_str(), 0, head, /*prefix=*/ false) ||
-1 == append_objects_from_xml_ex(prefix.c_str(), doc, ctx, ex_cprefix.c_str(), ex_prefix.c_str(), NULL, 1, head, /*prefix=*/ true) )
{
S3FS_PRN_ERR("append_objects_from_xml_ex returns with error.");
S3FS_XMLXPATHFREECONTEXT(ctx);

2
src/s3fs_xml.h
View File

@ -33,7 +33,7 @@ class S3ObjList;
// Functions
//-------------------------------------------------------------------
bool is_truncated(xmlDocPtr doc);
int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head);
int append_objects_from_xml_ex(const char* path, xmlDocPtr doc, xmlXPathContextPtr ctx, const char* ex_contents, const char* ex_key, const char* ex_etag, int isCPrefix, S3ObjList& head, bool prefix);
int append_objects_from_xml(const char* path, xmlDocPtr doc, S3ObjList& head);
xmlChar* get_next_continuation_token(xmlDocPtr doc);
xmlChar* get_next_marker(xmlDocPtr doc);

3
src/s3objlist.h
View File

@ -24,6 +24,7 @@
#include <list>
#include <map>
#include <string>
#include <vector>
//-------------------------------------------------------------------
// Structure / Typedef
@ -47,6 +48,8 @@ class S3ObjList
{
private:
s3obj_t objects;
public:
std::vector<std::string> common_prefixes;
private:
bool insert_normalized(const char* name, const char* normalized, bool is_dir);

194
test/integration-test-main.sh
View File

@ -389,6 +389,8 @@ function test_external_directory_creation {
describe "Test external directory creation ..."
local OBJECT_NAME; OBJECT_NAME=$(basename "${PWD}")/directory/"${TEST_TEXT_FILE}"
echo "data" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
# shellcheck disable=SC2010
ls | grep -q directory
ls directory >/dev/null 2>&1
get_permissions directory | grep -q 750$
ls directory
@ -2252,142 +2254,64 @@ function test_not_existed_dir_obj() {
echo data1 | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME_1}"
echo data2 | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME_2}"
# shellcheck disable=SC2009
if ps u -p "${S3FS_PID}" | grep -q compat_dir; then
#
# with "compat_dir", found directories and files
#
# Top directory
# shellcheck disable=SC2010
if ! ls -1 | grep -q '^not_existed_dir_single$'; then
echo "Expect to find \"not_existed_dir_single\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 | grep -q '^not_existed_dir_parent$'; then
echo "Expect to find \"not_existed_dir_parent\" directory, but it is not found"
return 1;
fi
# Single nest directory
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_single | grep -q '^not_existed_dir_single$'; then
echo "Expect to find \"not_existed_dir_single\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 not_existed_dir_single | grep -q "^${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_single/${TEST_TEXT_FILE}\" file, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 "not_existed_dir_single/${TEST_TEXT_FILE}" | grep -q "^not_existed_dir_single/${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_single/${TEST_TEXT_FILE}\" file, but it is not found"
return 1;
fi
# Double nest directory
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_parent | grep -q '^not_existed_dir_parent'; then
echo "Expect to find \"not_existed_dir_parent\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 not_existed_dir_parent | grep -q '^not_existed_dir_child'; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_parent/not_existed_dir_child | grep -q '^not_existed_dir_parent/not_existed_dir_child'; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 not_existed_dir_parent/not_existed_dir_child | grep -q "^${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 "not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}" | grep -q "^not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\" directory, but it is not found"
return 1;
fi
rm -rf not_existed_dir_single
rm -rf not_existed_dir_parent
else
#
# without "compat_dir", found directories and files
#
# [NOTE]
# If specify a directory path, the file under that directory will be found.
# And if specify a file full path, it will be found.
#
# Top directory
# shellcheck disable=SC2010
if ls -1 | grep -q '^not_existed_dir_single$'; then
echo "Expect to not find \"not_existed_dir_single\" directory, but it is found"
return 1;
fi
# shellcheck disable=SC2010
if ls -1 | grep -q '^not_existed_dir_parent$'; then
echo "Expect to not find \"not_existed_dir_parent\" directory, but it is found"
return 1;
fi
# Single nest directory
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_single | grep -q '^not_existed_dir_single$'; then
echo "Expect to find \"not_existed_dir_single\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 not_existed_dir_single | grep -q "^${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_single/${TEST_TEXT_FILE}\" file, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 "not_existed_dir_single/${TEST_TEXT_FILE}" | grep -q "^not_existed_dir_single/${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_single/${TEST_TEXT_FILE}\" file, but it is not found"
return 1;
fi
# Double nest directory
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_parent | grep -q '^not_existed_dir_parent'; then
echo "Expect to find \"not_existed_dir_parent\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ls -1 not_existed_dir_parent | grep -q '^not_existed_dir_child'; then
echo "Expect to not find \"not_existed_dir_parent/not_existed_dir_child\" directory, but it is found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_parent/not_existed_dir_child | grep -q '^not_existed_dir_parent/not_existed_dir_child'; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 not_existed_dir_parent/not_existed_dir_child | grep -q "^${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 "not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}" | grep -q "^not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\" directory, but it is not found"
return 1;
fi
rm -rf not_existed_dir_single
# [NOTE]
# This case could not remove sub directory, then below command will be failed.
#rm -rf not_existed_dir_parent
# Top directory
# shellcheck disable=SC2010
if ! ls -1 | grep -q '^not_existed_dir_single$'; then
echo "Expect to find \"not_existed_dir_single\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 | grep -q '^not_existed_dir_parent$'; then
echo "Expect to find \"not_existed_dir_parent\" directory, but it is not found"
return 1;
fi
# Single nest directory
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_single | grep -q '^not_existed_dir_single$'; then
echo "Expect to find \"not_existed_dir_single\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 not_existed_dir_single | grep -q "^${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_single/${TEST_TEXT_FILE}\" file, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 "not_existed_dir_single/${TEST_TEXT_FILE}" | grep -q "^not_existed_dir_single/${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_single/${TEST_TEXT_FILE}\" file, but it is not found"
return 1;
fi
# Double nest directory
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_parent | grep -q '^not_existed_dir_parent'; then
echo "Expect to find \"not_existed_dir_parent\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 not_existed_dir_parent | grep -q '^not_existed_dir_child'; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -d not_existed_dir_parent/not_existed_dir_child | grep -q '^not_existed_dir_parent/not_existed_dir_child'; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 not_existed_dir_parent/not_existed_dir_child | grep -q "^${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\" directory, but it is not found"
return 1;
fi
# shellcheck disable=SC2010
if ! ls -1 "not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}" | grep -q "^not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\$"; then
echo "Expect to find \"not_existed_dir_parent/not_existed_dir_child/${TEST_TEXT_FILE}\" directory, but it is not found"
return 1;
fi
rm -rf not_existed_dir_single
rm -rf not_existed_dir_parent
}
function test_ut_ossfs {