octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-06-06 11:00:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed a bug that padded null bytes when changing xattr

Browse Source
This commit is contained in:
Takeshi Nakatani 2021-02-20 17:03:41 +00:00 committed by Andrew Gaul
parent 81e267d421
commit 5b5bc3114a
3 changed files with 72 additions and 70 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/fdcache_entity.cpp
View File

@ -1581,7 +1581,7 @@ bool FdEntity::MergeOrgMeta(headers_t& updatemeta)
}
// global function in s3fs.cpp
int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mtime);
int put_headers(const char* path, headers_t& meta, bool is_copy);
int FdEntity::UploadPendingMeta()
{
@ -1593,7 +1593,7 @@ int FdEntity::UploadPendingMeta()
headers_t updatemeta = orgmeta;
updatemeta["x-amz-copy-source"] = urlEncode(service_path + bucket + get_realpath(path.c_str()));
// put headers, no need to update mtime to avoid dead lock
int result = put_headers(path.c_str(), updatemeta, true, false);
int result = put_headers(path.c_str(), updatemeta, true);
if(0 != result){
S3FS_PRN_ERR("failed to put header after flushing file(%s) by(%d).", path.c_str(), result);
}

94
src/s3fs.cpp
View File

@ -107,7 +107,7 @@ static const std::string aws_secretkey = "AWSSecretKey";
//-------------------------------------------------------------------
// Global functions : prototype
//-------------------------------------------------------------------
int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mtime = true); // [NOTE] global function because this is called from FdEntity class
int put_headers(const char* path, headers_t& meta, bool is_copy); // [NOTE] global function because this is called from FdEntity class
//-------------------------------------------------------------------
// Static functions : prototype
@ -737,7 +737,7 @@ static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char*
// ow_sse_flg is for over writing sse header by use_sse option.
// @return fuse return code
//
int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mtime)
int put_headers(const char* path, headers_t& meta, bool is_copy)
{
int result;
S3fsCurl s3fscurl(true);
@ -759,37 +759,6 @@ int put_headers(const char* path, headers_t& meta, bool is_copy, bool update_mti
return result;
}
}
// [NOTE]
// if path is 'dir/', it does not have cache(could not open file for directory stat)
//
if(update_mtime && '/' != path[strlen(path) - 1] ){
AutoFdEntity autoent;
FdEntity* ent;
if(NULL == (ent = autoent.ExistOpen(path, -1, !FdManager::IsCacheDir()))){
// no opened fd
if(FdManager::IsCacheDir()){
// create cache file if be needed
ent = autoent.Open(path, &meta, buf.st_size, -1, false, true);
}
}
if(ent){
time_t mtime = get_mtime(meta);
time_t ctime = get_ctime(meta);
time_t atime = get_atime(meta);
if(mtime < 0){
mtime = 0L;
}
if(ctime < 0){
ctime = 0L;
}
if(atime < 0){
atime = 0L;
}
ent->SetMCtime(mtime, ctime);
ent->SetAtime(atime);
}
}
return 0;
}
@ -1294,7 +1263,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
}
// copy
if(0 != (result = put_headers(to, meta, true, false))){
if(0 != (result = put_headers(to, meta, true))){
return result;
}
@ -1694,7 +1663,7 @@ static int s3fs_chmod(const char* _path, mode_t mode)
merge_headers(meta, updatemeta, true);
// upload meta directly.
if(0 != (result = put_headers(strpath.c_str(), meta, true, false))){
if(0 != (result = put_headers(strpath.c_str(), meta, true))){
return result;
}
StatCache::getStatCacheData()->DelStat(nowcache);
@ -1867,7 +1836,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
merge_headers(meta, updatemeta, true);
// upload meta directly.
if(0 != (result = put_headers(strpath.c_str(), meta, true, false))){
if(0 != (result = put_headers(strpath.c_str(), meta, true))){
return result;
}
StatCache::getStatCacheData()->DelStat(nowcache);
@ -2055,7 +2024,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2])
merge_headers(meta, updatemeta, true);
// upload meta directly.
if(0 != (result = put_headers(strpath.c_str(), meta, true, false))){
if(0 != (result = put_headers(strpath.c_str(), meta, true))){
return result;
}
StatCache::getStatCacheData()->DelStat(nowcache);
@ -3017,9 +2986,8 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value,
//
AutoFdEntity autoent;
FdEntity* ent;
bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){
// the file is opened now.
// get xattr and make new xattr
std::string strxattr;
if(ent->GetXattr(strxattr)){
@ -3035,31 +3003,26 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value,
}
if(ent->MergeOrgMeta(updatemeta)){
// now uploading
// the meta is pending and accumulated to be put after the upload is complete.
// meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete.
S3FS_PRN_INFO("meta pending until upload is complete");
}else{
// allow to put header
// updatemeta already merged the orgmeta of the opened files.
if(0 != (result = put_headers(strpath.c_str(), updatemeta, true))){
return result;
}
StatCache::getStatCacheData()->DelStat(nowcache);
need_put_header = false;
}
}else{
// not opened file, then put headers
merge_headers(meta, updatemeta, true);
// NOTICE: modify xattr from base meta
}
if(need_put_header){
// not found opened file.
if(0 != (result = set_xattrs_to_header(meta, name, value, size, flags))){
return result;
}
merge_headers(meta, updatemeta, true);
// upload meta directly.
if(0 != (result = put_headers(strpath.c_str(), meta, true))){
return result;
}
StatCache::getStatCacheData()->DelStat(nowcache);
}
return 0;
}
@ -3307,29 +3270,24 @@ static int s3fs_removexattr(const char* path, const char* name)
//
AutoFdEntity autoent;
FdEntity* ent;
bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){
// the file is opened now.
if(ent->MergeOrgMeta(updatemeta)){
// now uploading
// the meta is pending and accumulated to be put after the upload is complete.
// meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete.
S3FS_PRN_INFO("meta pending until upload is complete");
}else{
// allow to put header
// updatemeta already merged the orgmeta of the opened files.
if(updatemeta["x-amz-meta-xattr"].empty()){
updatemeta.erase("x-amz-meta-xattr");
}
if(0 != (result = put_headers(strpath.c_str(), updatemeta, true))){
return result;
}
StatCache::getStatCacheData()->DelStat(nowcache);
need_put_header = false;
}
}else{
// not opened file, then put headers
}
if(need_put_header){
// not found opened file.
if(updatemeta["x-amz-meta-xattr"].empty()){
updatemeta.erase("x-amz-meta-xattr");
}
merge_headers(meta, updatemeta, true);
// upload meta directly.
if(0 != (result = put_headers(strpath.c_str(), meta, true))){
return result;
}

44
test/integration-test-main.sh
View File

@ -354,6 +354,8 @@ function test_update_metadata_external_small_object() {
TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}"
TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}"
TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}"
TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
TEST_INPUT="TEST_STRING_IN_SMALL_FILE"
@ -381,9 +383,29 @@ function test_update_metadata_external_small_object() {
touch ${TEST_UTIMENS_FILE}
cmp ${TEST_UTIMENS_FILE} <(echo "${TEST_INPUT}")
#
# set xattr
#
OBJECT_NAME="$(basename $PWD)/${TEST_SETXATTR_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}"
set_xattr key value ${TEST_SETXATTR_FILE}
cmp ${TEST_SETXATTR_FILE} <(echo "${TEST_INPUT}")
#
# remove xattr
#
# "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"}
#
OBJECT_NAME="$(basename $PWD)/${TEST_RMXATTR_FILE}"
echo "${TEST_INPUT}" | aws_cli s3 cp - "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D
del_xattr key ${TEST_RMXATTR_FILE}
cmp ${TEST_RMXATTR_FILE} <(echo "${TEST_INPUT}")
rm -f ${TEST_CHMOD_FILE}
rm -f ${TEST_CHOWN_FILE}
rm -f ${TEST_UTIMENS_FILE}
rm -f ${TEST_SETXATTR_FILE}
rm -f ${TEST_RMXATTR_FILE}
}
function test_update_metadata_external_large_object() {
@ -396,6 +418,8 @@ function test_update_metadata_external_large_object() {
TEST_CHMOD_FILE="${TEST_TEXT_FILE}_chmod.${TEST_FILE_EXT}"
TEST_CHOWN_FILE="${TEST_TEXT_FILE}_chown.${TEST_FILE_EXT}"
TEST_UTIMENS_FILE="${TEST_TEXT_FILE}_utimens.${TEST_FILE_EXT}"
TEST_SETXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
TEST_RMXATTR_FILE="${TEST_TEXT_FILE}_xattr.${TEST_FILE_EXT}"
dd if=/dev/urandom of="${TEMP_DIR}/${BIG_FILE}" bs=$BIG_FILE_BLOCK_SIZE count=$BIG_FILE_COUNT
@ -423,10 +447,30 @@ function test_update_metadata_external_large_object() {
touch ${TEST_UTIMENS_FILE}
cmp ${TEST_UTIMENS_FILE} ${TEMP_DIR}/${BIG_FILE}
#
# set xattr
#
OBJECT_NAME="$(basename $PWD)/${TEST_SETXATTR_FILE}"
aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress
set_xattr key value ${TEST_SETXATTR_FILE}
cmp ${TEST_SETXATTR_FILE} ${TEMP_DIR}/${BIG_FILE}
#
# remove xattr
#
# "%7B%22key%22%3A%22dmFsdWU%3D%22%7D" = {"key":"value"}
#
OBJECT_NAME="$(basename $PWD)/${TEST_RMXATTR_FILE}"
aws_cli s3 cp ${TEMP_DIR}/${BIG_FILE} "s3://${TEST_BUCKET_1}/${OBJECT_NAME}" --no-progress --metadata xattr=%7B%22key%22%3A%22dmFsdWU%3D%22%7D
del_xattr key ${TEST_RMXATTR_FILE}
cmp ${TEST_RMXATTR_FILE} ${TEMP_DIR}/${BIG_FILE}
rm -f ${TEMP_DIR}/${BIG_FILE}
rm -f ${TEST_CHMOD_FILE}
rm -f ${TEST_CHOWN_FILE}
rm -f ${TEST_UTIMENS_FILE}
rm -f ${TEST_SETXATTR_FILE}
rm -f ${TEST_RMXATTR_FILE}
}
function test_rename_before_close {