octoleo
/
s3fs-fuse
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed a bug about truncation for shrinking file

This commit is contained in:
Takeshi Nakatani 2022-02-27 06:34:45 +00:00 committed by Andrew Gaul
parent 684ced5a41
commit d842d45b2b
8 changed files with 109 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
src/fdcache.cpp
View File

@ -409,6 +409,16 @@ bool FdManager::HasOpenEntityFd(const char* path)
return (0 < ent->GetOpenCount());
}
// [NOTE]
// Returns the number of open pseudo fd.
//
int FdManager::GetOpenFdCount(const char* path)
{
AutoLock auto_lock(&FdManager::fd_manager_lock);
return FdManager::singleton.GetPseudoFdCount(path);
}
//------------------------------------------------
// FdManager methods
//------------------------------------------------
@ -523,9 +533,9 @@ FdEntity* FdManager::GetFdEntity(const char* path, int& existfd, bool newfd, boo
return NULL;
}
FdEntity* FdManager::Open(int& fd, const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, AutoLock::Type type)
FdEntity* FdManager::Open(int& fd, const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, bool ignore_modify, AutoLock::Type type)
{
S3FS_PRN_DBG("[path=%s][size=%lld][time=%lld][flags=0x%x]", SAFESTRPTR(path), static_cast<long long>(size), static_cast<long long>(time), flags);
S3FS_PRN_DBG("[path=%s][size=%lld][time=%lld][flags=0x%x][force_tmpfile=%s][create=%s][ignore_modify=%s]", SAFESTRPTR(path), static_cast<long long>(size), static_cast<long long>(time), flags, (force_tmpfile ? "yes" : "no"), (is_create ? "yes" : "no"), (ignore_modify ? "yes" : "no"));
if(!path || '\0' == path[0]){
return NULL;
@ -554,11 +564,12 @@ FdEntity* FdManager::Open(int& fd, const char* path, headers_t* pmeta, off_t siz
ent = iter->second;
// [NOTE]
// Even if getting the request to change the size of modifying
// file to small, we do not change it. Because if it will change,
// test_open_second_fd test will be failed.
// If the file is being modified and ignore_modify flag is false,
// the file size will not be changed even if there is a request
// to reduce the size of the modified file.
// If you do, the "test_open_second_fd" test will fail.
//
if(ent->IsModified()){
if(!ignore_modify && ent->IsModified()){
// If the file is being modified and it's size is larger than size parameter, it will not be resized.
off_t cur_size = 0;
if(ent->GetSize(cur_size) && size <= cur_size){
@ -635,7 +646,7 @@ FdEntity* FdManager::OpenExistFdEntity(const char* path, int& fd, int flags)
S3FS_PRN_DBG("[path=%s][flags=0x%x]", SAFESTRPTR(path), flags);
// search entity by path, and create pseudo fd
FdEntity* ent = Open(fd, path, NULL, -1, -1, flags, false, false, AutoLock::NONE);
FdEntity* ent = Open(fd, path, NULL, -1, -1, flags, false, false, false, AutoLock::NONE);
if(!ent){
// Not found entity
return NULL;
@ -643,6 +654,29 @@ FdEntity* FdManager::OpenExistFdEntity(const char* path, int& fd, int flags)
return ent;
}
// [NOTE]
// Returns the number of open pseudo fd.
// This method is called from GetOpenFdCount method which is already locked.
//
int FdManager::GetPseudoFdCount(const char* path)
{
S3FS_PRN_DBG("[path=%s]", SAFESTRPTR(path));
if(!path || '\0' == path[0]){
return 0;
}
// search from all entity.
for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){
if(iter->second && 0 == strcmp(iter->second->GetPath(), path)){
// found the entity for the path
return iter->second->GetOpenCount();
}
}
// not found entity
return 0;
}
void FdManager::Rename(const std::string &from, const std::string &to)
{
AutoLock auto_lock(&FdManager::fd_manager_lock);

7
src/fdcache.h
View File

@ -47,9 +47,11 @@ class FdManager
private:
static off_t GetFreeDiskSpace(const char* path);
static bool IsDir(const std::string* dir);
int GetPseudoFdCount(const char* path);
void CleanupCacheDirInternal(const std::string &path = "");
bool RawCheckAllCache(FILE* fp, const char* cache_stat_top_dir, const char* sub_path, int& total_file_cnt, int& err_file_cnt, int& err_dir_cnt);
static bool IsDir(const std::string* dir);
public:
FdManager();
@ -71,6 +73,7 @@ class FdManager
static bool SetCheckCacheDirExist(bool is_check);
static bool CheckCacheDirExist();
static bool HasOpenEntityFd(const char* path);
static int GetOpenFdCount(const char* path);
static off_t GetEnsureFreeDiskSpace();
static off_t SetEnsureFreeDiskSpace(off_t size);
static bool InitFakeUsedDiskSize(off_t fake_freesize);
@ -84,7 +87,7 @@ class FdManager
// Return FdEntity associated with path, returning NULL on error. This operation increments the reference count; callers must decrement via Close after use.
FdEntity* GetFdEntity(const char* path, int& existfd, bool newfd = true, bool lock_already_held = false);
FdEntity* Open(int& fd, const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, AutoLock::Type type);
FdEntity* Open(int& fd, const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, bool ignore_modify, AutoLock::Type type);
FdEntity* GetExistFdEntity(const char* path, int existfd = -1);
FdEntity* OpenExistFdEntity(const char* path, int& fd, int flags = O_RDONLY);
void Rename(const std::string &from, const std::string &to);

4
src/fdcache_auto.cpp
View File

@ -98,11 +98,11 @@ bool AutoFdEntity::Attach(const char* path, int existfd)
return true;
}
FdEntity* AutoFdEntity::Open(const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, AutoLock::Type type)
FdEntity* AutoFdEntity::Open(const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, bool ignore_modify, AutoLock::Type type)
{
Close();
if(NULL == (pFdEntity = FdManager::get()->Open(pseudo_fd, path, pmeta, size, time, flags, force_tmpfile, is_create, type))){
if(NULL == (pFdEntity = FdManager::get()->Open(pseudo_fd, path, pmeta, size, time, flags, force_tmpfile, is_create, ignore_modify, type))){
pseudo_fd = -1;
return NULL;
}

2
src/fdcache_auto.h
View File

@ -51,7 +51,7 @@ class AutoFdEntity
bool Attach(const char* path, int existfd);
int GetPseudoFd() const { return pseudo_fd; }
FdEntity* Open(const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, AutoLock::Type type);
FdEntity* Open(const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, bool ignore_modify, AutoLock::Type type);
FdEntity* GetExistFdEntity(const char* path, int existfd = -1);
FdEntity* OpenExistFdEntity(const char* path, int flags = O_RDONLY);
};

14
src/fdcache_page.cpp
View File

@ -355,7 +355,7 @@ void PageList::FreeList(fdpage_list_t& list)
list.clear();
}
PageList::PageList(off_t size, bool is_loaded, bool is_modified)
PageList::PageList(off_t size, bool is_loaded, bool is_modified, bool shrinked) : is_shrink(shrinked)
{
Init(size, is_loaded, is_modified);
}
@ -365,6 +365,7 @@ PageList::PageList(const PageList& other)
for(fdpage_list_t::const_iterator iter = other.pages.begin(); iter != other.pages.end(); ++iter){
pages.push_back(*iter);
}
is_shrink = other.is_shrink;
}
PageList::~PageList()
@ -375,6 +376,7 @@ PageList::~PageList()
void PageList::Clear()
{
PageList::FreeList(pages);
is_shrink = false;
}
bool PageList::Init(off_t size, bool is_loaded, bool is_modified)
@ -444,6 +446,9 @@ bool PageList::Resize(off_t size, bool is_loaded, bool is_modified)
}
}
}
if(is_modified){
is_shrink = true;
}
}else{ // total == size
// nothing to do
}
@ -766,6 +771,9 @@ off_t PageList::BytesModified() const
bool PageList::IsModified() const
{
if(is_shrink){
return true;
}
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter){
if(iter->modified){
return true;
@ -776,6 +784,8 @@ bool PageList::IsModified() const
bool PageList::ClearAllModified()
{
is_shrink = false;
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); ++iter){
if(iter->modified){
iter->modified = false;
@ -939,7 +949,7 @@ void PageList::Dump() const
{
int cnt = 0;
S3FS_PRN_DBG("pages = {");
S3FS_PRN_DBG("pages (shrinked=%s) = {", (is_shrink ? "yes" : "no"));
for(fdpage_list_t::const_iterator iter = pages.begin(); iter != pages.end(); ++iter, ++cnt){
S3FS_PRN_DBG(" [%08d] -> {%014lld - %014lld : %s / %s}", cnt, static_cast<long long int>(iter->offset), static_cast<long long int>(iter->bytes), iter->loaded ? "loaded" : "unloaded", iter->modified ? "modified" : "not modified");
}

3
src/fdcache_page.h
View File

@ -77,6 +77,7 @@ class PageList
private:
fdpage_list_t pages;
bool is_shrink; // [NOTE] true if it has been shrinked even once
public:
enum page_status{
@ -97,7 +98,7 @@ class PageList
public:
static void FreeList(fdpage_list_t& list);
explicit PageList(off_t size = 0, bool is_loaded = false, bool is_modified = false);
explicit PageList(off_t size = 0, bool is_loaded = false, bool is_modified = false, bool shrinked = false);
explicit PageList(const PageList& other);
~PageList();

32
src/s3fs.cpp
View File

@ -705,7 +705,7 @@ static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char*
time_t mtime = (!S_ISREG(stobj.st_mode) && !S_ISLNK(stobj.st_mode)) ? -1 : stobj.st_mtime;
bool force_tmpfile = S_ISREG(stobj.st_mode) ? false : true;
if(NULL == (ent = autoent.Open(path, &meta, stobj.st_size, mtime, flags, force_tmpfile, true, AutoLock::NONE))){
if(NULL == (ent = autoent.Open(path, &meta, stobj.st_size, mtime, flags, force_tmpfile, true, false, AutoLock::NONE))){
S3FS_PRN_ERR("Could not open file. errno(%d)", errno);
return -EIO;
}
@ -939,7 +939,7 @@ static int s3fs_create(const char* _path, mode_t mode, struct fuse_file_info* fi
AutoFdEntity autoent;
FdEntity* ent;
if(NULL == (ent = autoent.Open(path, &meta, 0, -1, fi->flags, false, true, AutoLock::NONE))){
if(NULL == (ent = autoent.Open(path, &meta, 0, -1, fi->flags, false, true, false, AutoLock::NONE))){
StatCache::getStatCacheData()->DelStat(path);
return -EIO;
}
@ -1135,7 +1135,7 @@ static int s3fs_symlink(const char* _from, const char* _to)
{ // scope for AutoFdEntity
AutoFdEntity autoent;
FdEntity* ent;
if(NULL == (ent = autoent.Open(to, &headers, 0, -1, O_RDWR, true, true, AutoLock::NONE))){
if(NULL == (ent = autoent.Open(to, &headers, 0, -1, O_RDWR, true, true, false, AutoLock::NONE))){
S3FS_PRN_ERR("could not open tmpfile(errno=%d)", errno);
return -errno;
}
@ -1208,7 +1208,7 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
// no opened fd
if(FdManager::IsCacheDir()){
// create cache file if be needed
ent = autoent.Open(from, &meta, buf.st_size, -1, O_RDONLY, false, true, AutoLock::NONE);
ent = autoent.Open(from, &meta, buf.st_size, -1, O_RDONLY, false, true, false, AutoLock::NONE);
}
if(ent){
struct timespec mtime = get_mtime(meta);
@ -2178,7 +2178,25 @@ static int s3fs_truncate(const char* _path, off_t size)
// The Flush is called before this file is closed, so there is no need
// to do it here.
//
if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, false, true, AutoLock::NONE))){
// [NOTICE]
// FdManager::Open() ignores changes that reduce the file size for the
// file you are editing. However, if user opens only onece, edit it,
// and then shrink the file, it should be done.
// When this function is called, the file is already open by FUSE or
// some other operation. Therefore, if the number of open files is 1,
// no edits other than that fd will be made, and the files will be
// shrunk using ignore_modify flag even while editing.
// See the comments when merging this code for FUSE2 limitations.
// (In FUSE3, it will be possible to handle it reliably using fuse_file_info.)
//
bool ignore_modify;
if(1 < FdManager::GetOpenFdCount(path)){
ignore_modify = false;
}else{
ignore_modify = true;
}
if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, false, true, ignore_modify, AutoLock::NONE))){
S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno);
return -EIO;
}
@ -2212,7 +2230,7 @@ static int s3fs_truncate(const char* _path, off_t size)
meta["x-amz-meta-uid"] = str(pcxt->uid);
meta["x-amz-meta-gid"] = str(pcxt->gid);
if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, true, true, AutoLock::NONE))){
if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, true, true, false, AutoLock::NONE))){
S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno);
return -EIO;
}
@ -2280,7 +2298,7 @@ static int s3fs_open(const char* _path, struct fuse_file_info* fi)
FdEntity* ent;
headers_t meta;
get_object_attribute(path, NULL, &meta, true, NULL, true); // no truncate cache
if(NULL == (ent = autoent.Open(path, &meta, st.st_size, st.st_mtime, fi->flags, false, true, AutoLock::NONE))){
if(NULL == (ent = autoent.Open(path, &meta, st.st_size, st.st_mtime, fi->flags, false, true, false, AutoLock::NONE))){
StatCache::getStatCacheData()->DelStat(path);
return -EIO;
}

21
test/integration-test-main.sh
View File

@ -92,6 +92,26 @@ function test_truncate_empty_file {
rm_test_file
}
function test_truncate_shrink_file {
describe "Testing truncate shrinking large binary file ..."
local BIG_TRUNCATE_TEST_FILE="big-truncate-test.bin"
local t_size=$((1024 * 1024 * 32 + 64))
dd if=/dev/urandom of="${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" bs=1024 count=$((1024 * 64))
cp "${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" "${BIG_TRUNCATE_TEST_FILE}"
"${TRUNCATE_BIN}" "${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" -s "${t_size}"
"${TRUNCATE_BIN}" "${BIG_TRUNCATE_TEST_FILE}" -s "${t_size}"
if ! cmp "${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}" "${BIG_TRUNCATE_TEST_FILE}"; then
return 1
fi
rm -f "${TEMP_DIR}/${BIG_TRUNCATE_TEST_FILE}"
rm_test_file "${BIG_TRUNCATE_TEST_FILE}"
}
function test_mv_file {
describe "Testing mv file function ..."
# if the rename file exists, delete it
@ -1818,6 +1838,7 @@ function add_all_tests {
add_tests test_truncate_file
add_tests test_truncate_upload
add_tests test_truncate_empty_file
add_tests test_truncate_shrink_file
add_tests test_mv_file
add_tests test_mv_to_exist_file
add_tests test_mv_empty_directory