octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-11-14 16:34:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introduced pseudo fd and separated fd for each file opening

Browse Source
This commit is contained in:
Takeshi Nakatani 2021-05-23 16:28:50 +00:00 committed by Andrew Gaul
parent 53dfd48f59
commit ac578d188e
14 changed files with 1068 additions and 573 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Makefile.am
View File

@ -47,6 +47,8 @@ s3fs_SOURCES = \
fdcache_page.cpp \ fdcache_page.cpp \
fdcache_stat.cpp \ fdcache_stat.cpp \
fdcache_auto.cpp \ fdcache_auto.cpp \
fdcache_fdinfo.cpp \
fdcache_pseudofd.cpp \
addhead.cpp \ addhead.cpp \
sighandlers.cpp \ sighandlers.cpp \
autolock.cpp \ autolock.cpp \

61
src/curl.cpp
View File

@ -3980,67 +3980,6 @@ int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& met
return 0; return 0;
} }
int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool is_copy)
{
int result;
std::string upload_id;
struct stat st;
int fd2;
etaglist_t list;
off_t remaining_bytes;
off_t chunk;
S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd);
// duplicate fd
if(-1 == (fd2 = dup(fd)) || 0 != lseek(fd2, 0, SEEK_SET)){
S3FS_PRN_ERR("Could not duplicate file descriptor(errno=%d)", errno);
if(-1 != fd2){
close(fd2);
}
return -errno;
}
if(-1 == fstat(fd2, &st)){
S3FS_PRN_ERR("Invalid file descriptor(errno=%d)", errno);
close(fd2);
return -errno;
}
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, is_copy))){
close(fd2);
return result;
}
DestroyCurlHandle();
// cycle through open fd, pulling off 10MB chunks at a time
for(remaining_bytes = st.st_size; 0 < remaining_bytes; remaining_bytes -= chunk){
// chunk size
chunk = remaining_bytes > S3fsCurl::multipart_size ? S3fsCurl::multipart_size : remaining_bytes;
// set
partdata.fd = fd2;
partdata.startpos = st.st_size - remaining_bytes;
partdata.size = chunk;
b_partdata_startpos = partdata.startpos;
b_partdata_size = partdata.size;
partdata.add_etag_list(&list);
// upload part
if(0 != (result = UploadMultipartPostRequest(tpath, list.size(), upload_id))){
S3FS_PRN_ERR("failed uploading part(%d)", result);
close(fd2);
return result;
}
DestroyCurlHandle();
}
close(fd2);
if(0 != (result = CompleteMultipartPostRequest(tpath, upload_id, list))){
return result;
}
return 0;
}
int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etaglist_t& list) int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etaglist_t& list)
{ {
S3FS_PRN_INFO3("[upload_id=%s][tpath=%s][fd=%d][offset=%lld][size=%lld]", upload_id.c_str(), SAFESTRPTR(tpath), fd, static_cast<long long int>(offset), static_cast<long long int>(size)); S3FS_PRN_INFO3("[upload_id=%s][tpath=%s][fd=%d][offset=%lld][size=%lld]", upload_id.c_str(), SAFESTRPTR(tpath), fd, static_cast<long long int>(offset), static_cast<long long int>(size));

1
src/curl.h
View File

@ -393,7 +393,6 @@ class S3fsCurl
int MultipartListRequest(std::string& body); int MultipartListRequest(std::string& body);
int AbortMultipartUpload(const char* tpath, const std::string& upload_id); int AbortMultipartUpload(const char* tpath, const std::string& upload_id);
int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy); int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy);
int MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool is_copy);
int MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etaglist_t& list); int MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etaglist_t& list);
int MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size); int MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size);

149
src/fdcache.cpp
View File

@ -28,6 +28,7 @@
#include "common.h" #include "common.h"
#include "s3fs.h" #include "s3fs.h"
#include "fdcache.h" #include "fdcache.h"
#include "fdcache_pseudofd.h"
#include "s3fs_util.h" #include "s3fs_util.h"
#include "s3fs_logger.h" #include "s3fs_logger.h"
#include "string_util.h" #include "string_util.h"
@ -344,6 +345,16 @@ bool FdManager::HaveLseekHole()
return FdManager::have_lseek_hole; return FdManager::have_lseek_hole;
} }
bool FdManager::HasOpenEntityFd(const char* path)
{
FdEntity* ent;
int fd = -1;
if(NULL == (ent = FdManager::singleton.GetFdEntity(path, fd, false))){
return false;
}
return (0 < ent->GetOpenCount());
}
//------------------------------------------------ //------------------------------------------------
// FdManager methods // FdManager methods
//------------------------------------------------ //------------------------------------------------
@ -379,7 +390,7 @@ FdManager::~FdManager()
if(this == FdManager::get()){ if(this == FdManager::get()){
for(fdent_map_t::iterator iter = fent.begin(); fent.end() != iter; ++iter){ for(fdent_map_t::iterator iter = fent.begin(); fent.end() != iter; ++iter){
FdEntity* ent = (*iter).second; FdEntity* ent = (*iter).second;
S3FS_PRN_WARN("To exit with the cache file opened: path=%s, refcnt=%d", ent->GetPath(), ent->GetRefCnt()); S3FS_PRN_WARN("To exit with the cache file opened: path=%s, refcnt=%d", ent->GetPath(), ent->GetOpenCount());
delete ent; delete ent;
} }
fent.clear(); fent.clear();
@ -405,7 +416,7 @@ FdManager::~FdManager()
} }
} }
FdEntity* FdManager::GetFdEntity(const char* path, int existfd, bool increase_ref) FdEntity* FdManager::GetFdEntity(const char* path, int& existfd, bool newfd)
{ {
S3FS_PRN_INFO3("[path=%s][fd=%d]", SAFESTRPTR(path), existfd); S3FS_PRN_INFO3("[path=%s][fd=%d]", SAFESTRPTR(path), existfd);
@ -415,22 +426,29 @@ FdEntity* FdManager::GetFdEntity(const char* path, int existfd, bool increase_re
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
fdent_map_t::iterator iter = fent.find(std::string(path)); fdent_map_t::iterator iter = fent.find(std::string(path));
if(fent.end() != iter && (-1 == existfd || (*iter).second->GetFd() == existfd)){ if(fent.end() != iter && iter->second){
if(increase_ref){ if(-1 == existfd){
iter->second->Dup(); if(newfd){
existfd = iter->second->OpenPseudoFd(O_RDWR); // [NOTE] O_RDWR flags
}
return iter->second;
}else if(iter->second->FindPseudoFd(existfd)){
if(newfd){
existfd = iter->second->Dup(existfd);
}
return iter->second;
} }
return (*iter).second;
} }
if(-1 != existfd){ if(-1 != existfd){
for(iter = fent.begin(); iter != fent.end(); ++iter){ for(iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second && (*iter).second->GetFd() == existfd){ if(iter->second && iter->second->FindPseudoFd(existfd)){
// found opened fd in map // found opened fd in map
if(0 == strcmp((*iter).second->GetPath(), path)){ if(0 == strcmp(iter->second->GetPath(), path)){
if(increase_ref){ if(newfd){
iter->second->Dup(); existfd = iter->second->Dup(existfd);
} }
return (*iter).second; return iter->second;
} }
// found fd, but it is used another file(file descriptor is recycled) // found fd, but it is used another file(file descriptor is recycled)
// so returns NULL. // so returns NULL.
@ -438,24 +456,30 @@ FdEntity* FdManager::GetFdEntity(const char* path, int existfd, bool increase_re
} }
} }
} }
// If the cache directory is not specified, s3fs opens a temporary file
// when the file is opened.
if(!FdManager::IsCacheDir()){
for(iter = fent.begin(); iter != fent.end(); ++iter){
if(iter->second && iter->second->IsOpen() && 0 == strcmp(iter->second->GetPath(), path)){
return iter->second;
}
}
}
return NULL; return NULL;
} }
FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t time, bool force_tmpfile, bool is_create, bool no_fd_lock_wait) FdEntity* FdManager::Open(int& fd, const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, bool no_fd_lock_wait)
{ {
S3FS_PRN_DBG("[path=%s][size=%lld][time=%lld]", SAFESTRPTR(path), static_cast<long long>(size), static_cast<long long>(time)); S3FS_PRN_DBG("[path=%s][size=%lld][time=%lld][flags=0x%x]", SAFESTRPTR(path), static_cast<long long>(size), static_cast<long long>(time), flags);
if(!path || '\0' == path[0]){ if(!path || '\0' == path[0]){
return NULL; return NULL;
} }
bool close = false;
FdEntity* ent;
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
// search in mapping by key(path) // search in mapping by key(path)
fdent_map_t::iterator iter = fent.find(std::string(path)); fdent_map_t::iterator iter = fent.find(std::string(path));
if(fent.end() == iter && !force_tmpfile && !FdManager::IsCacheDir()){ if(fent.end() == iter && !force_tmpfile && !FdManager::IsCacheDir()){
// If the cache directory is not specified, s3fs opens a temporary file // If the cache directory is not specified, s3fs opens a temporary file
// when the file is opened. // when the file is opened.
@ -463,16 +487,17 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t
// search a entity in all which opened the temporary file. // search a entity in all which opened the temporary file.
// //
for(iter = fent.begin(); iter != fent.end(); ++iter){ for(iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second && (*iter).second->IsOpen() && 0 == strcmp((*iter).second->GetPath(), path)){ if(iter->second && iter->second->IsOpen() && 0 == strcmp(iter->second->GetPath(), path)){
break; // found opened fd in mapping break; // found opened fd in mapping
} }
} }
} }
FdEntity* ent;
if(fent.end() != iter){ if(fent.end() != iter){
// found // found
ent = (*iter).second; ent = iter->second;
ent->Dup();
if(ent->IsModified()){ if(ent->IsModified()){
// If the file is being modified and it's size is larger than size parameter, it will not be resized. // If the file is being modified and it's size is larger than size parameter, it will not be resized.
off_t cur_size = 0; off_t cur_size = 0;
@ -480,7 +505,12 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t
size = -1; size = -1;
} }
} }
close = true;
// (re)open
if(-1 == (fd = ent->Open(pmeta, size, time, flags, no_fd_lock_wait ? AutoLock::NO_WAIT : AutoLock::NON))){
S3FS_PRN_ERR("failed to (re)open and create new pseudo fd for path(%s).", path);
return NULL;
}
}else if(is_create){ }else if(is_create){
// not found // not found
@ -492,6 +522,12 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t
// make new obj // make new obj
ent = new FdEntity(path, cache_path.c_str()); ent = new FdEntity(path, cache_path.c_str());
// open
if(-1 == (fd = ent->Open(pmeta, size, time, flags, no_fd_lock_wait ? AutoLock::NO_WAIT : AutoLock::NON))){
delete ent;
return NULL;
}
if(!cache_path.empty()){ if(!cache_path.empty()){
// using cache // using cache
fent[std::string(path)] = ent; fent[std::string(path)] = ent;
@ -507,47 +543,43 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t
FdManager::MakeRandomTempPath(path, tmppath); FdManager::MakeRandomTempPath(path, tmppath);
fent[tmppath] = ent; fent[tmppath] = ent;
} }
}else{ }else{
return NULL; return NULL;
} }
// open
if(0 != ent->Open(pmeta, size, time, no_fd_lock_wait ? AutoLock::NO_WAIT : AutoLock::NONE)){
if(close){
ent->Close();
}
return NULL;
}
if(close){
ent->Close();
}
return ent; return ent;
} }
FdEntity* FdManager::ExistOpen(const char* path, int existfd, bool ignore_existfd) // [NOTE]
// This method does not create a new pseudo fd.
// It just finds existfd and returns the corresponding entity.
//
FdEntity* FdManager::GetExistFdEntiy(const char* path, int existfd)
{ {
S3FS_PRN_DBG("[path=%s][fd=%d][ignore_existfd=%s]", SAFESTRPTR(path), existfd, ignore_existfd ? "true" : "false"); S3FS_PRN_DBG("[path=%s][existfd=%d]", SAFESTRPTR(path), existfd);
// search by real path
FdEntity* ent = Open(path, NULL, -1, -1, false, false);
if(!ent && (ignore_existfd || (-1 != existfd))){
// search from all fdentity because of not using cache.
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
// search from all entity.
for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){ for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second && (*iter).second->IsOpen() && (ignore_existfd || ((*iter).second->GetFd() == existfd))){ if(iter->second && iter->second->FindPseudoFd(existfd)){
// found opened fd in map // found existfd in entity
if(0 == strcmp((*iter).second->GetPath(), path)){ return iter->second;
ent = (*iter).second;
ent->Dup();
}else{
// found fd, but it is used another file(file descriptor is recycled)
// so returns NULL.
}
break;
} }
} }
// not found entity
return NULL;
}
FdEntity* FdManager::OpenExistFdEntiy(const char* path, int& fd, int flags)
{
S3FS_PRN_DBG("[path=%s][flags=0x%x]", SAFESTRPTR(path), flags);
// search entity by path, and create pseudo fd
FdEntity* ent = Open(fd, path, NULL, -1, -1, flags, false, false);
if(!ent){
// Not found entity
return NULL;
} }
return ent; return ent;
} }
@ -564,7 +596,7 @@ void FdManager::Rename(const std::string &from, const std::string &to)
// search a entity in all which opened the temporary file. // search a entity in all which opened the temporary file.
// //
for(iter = fent.begin(); iter != fent.end(); ++iter){ for(iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second && (*iter).second->IsOpen() && 0 == strcmp((*iter).second->GetPath(), from.c_str())){ if(iter->second && iter->second->IsOpen() && 0 == strcmp(iter->second->GetPath(), from.c_str())){
break; // found opened fd in mapping break; // found opened fd in mapping
} }
} }
@ -574,7 +606,7 @@ void FdManager::Rename(const std::string &from, const std::string &to)
// found // found
S3FS_PRN_DBG("[from=%s][to=%s]", from.c_str(), to.c_str()); S3FS_PRN_DBG("[from=%s][to=%s]", from.c_str(), to.c_str());
FdEntity* ent = (*iter).second; FdEntity* ent = iter->second;
// retrieve old fd entity from map // retrieve old fd entity from map
fent.erase(iter); fent.erase(iter);
@ -591,26 +623,25 @@ void FdManager::Rename(const std::string &from, const std::string &to)
} }
} }
bool FdManager::Close(FdEntity* ent) bool FdManager::Close(FdEntity* ent, int fd)
{ {
S3FS_PRN_DBG("[ent->file=%s][ent->fd=%d]", ent ? ent->GetPath() : "", ent ? ent->GetFd() : -1); S3FS_PRN_DBG("[ent->file=%s][pseudo_fd=%d]", ent ? ent->GetPath() : "", fd);
if(!ent){ if(!ent || -1 == fd){
return true; // returns success return true; // returns success
} }
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){ for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second == ent){ if(iter->second == ent){
ent->Close(); ent->Close(fd);
if(!ent->IsOpen()){ if(!ent->IsOpen()){
// remove found entity from map. // remove found entity from map.
fent.erase(iter++); fent.erase(iter++);
// check another key name for entity value to be on the safe side // check another key name for entity value to be on the safe side
for(; iter != fent.end(); ){ for(; iter != fent.end(); ){
if((*iter).second == ent){ if(iter->second == ent){
fent.erase(iter++); fent.erase(iter++);
}else{ }else{
++iter; ++iter;
@ -629,7 +660,7 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ){ for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ){
if((*iter).second == ent){ if(iter->second == ent){
fent.erase(iter++); fent.erase(iter++);
std::string tmppath; std::string tmppath;

11
src/fdcache.h
View File

@ -67,7 +67,7 @@ class FdManager
static bool MakeRandomTempPath(const char* path, std::string& tmppath); static bool MakeRandomTempPath(const char* path, std::string& tmppath);
static bool SetCheckCacheDirExist(bool is_check); static bool SetCheckCacheDirExist(bool is_check);
static bool CheckCacheDirExist(); static bool CheckCacheDirExist();
static bool HasOpenEntityFd(const char* path);
static off_t GetEnsureFreeDiskSpace(); static off_t GetEnsureFreeDiskSpace();
static off_t SetEnsureFreeDiskSpace(off_t size); static off_t SetEnsureFreeDiskSpace(off_t size);
static bool IsSafeDiskSpace(const char* path, off_t size); static bool IsSafeDiskSpace(const char* path, off_t size);
@ -76,11 +76,12 @@ class FdManager
static bool HaveLseekHole(); static bool HaveLseekHole();
// Return FdEntity associated with path, returning NULL on error. This operation increments the reference count; callers must decrement via Close after use. // Return FdEntity associated with path, returning NULL on error. This operation increments the reference count; callers must decrement via Close after use.
FdEntity* GetFdEntity(const char* path, int existfd = -1, bool increase_ref = true); FdEntity* GetFdEntity(const char* path, int& existfd, bool newfd = true);
FdEntity* Open(const char* path, headers_t* pmeta = NULL, off_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false); FdEntity* Open(int& fd, const char* path, headers_t* pmeta = NULL, off_t size = -1, time_t time = -1, int flags = O_RDONLY, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false);
FdEntity* ExistOpen(const char* path, int existfd = -1, bool ignore_existfd = false); FdEntity* GetExistFdEntiy(const char* path, int existfd = -1);
FdEntity* OpenExistFdEntiy(const char* path, int& fd, int flags = O_RDONLY);
void Rename(const std::string &from, const std::string &to); void Rename(const std::string &from, const std::string &to);
bool Close(FdEntity* ent); bool Close(FdEntity* ent, int fd);
bool ChangeEntityToTempPath(FdEntity* ent, const char* path); bool ChangeEntityToTempPath(FdEntity* ent, const char* path);
void CleanupCacheDir(); void CleanupCacheDir();

75
src/fdcache_auto.cpp
View File

@ -29,7 +29,7 @@
//------------------------------------------------ //------------------------------------------------
// AutoFdEntity methods // AutoFdEntity methods
//------------------------------------------------ //------------------------------------------------
AutoFdEntity::AutoFdEntity() : pFdEntity(NULL) AutoFdEntity::AutoFdEntity() : pFdEntity(NULL), pseudo_fd(-1)
{ {
} }
@ -38,13 +38,16 @@ AutoFdEntity::AutoFdEntity() : pFdEntity(NULL)
// Even if it is called, the consistency of the number of // Even if it is called, the consistency of the number of
// references can be maintained, but this case is not assumed. // references can be maintained, but this case is not assumed.
// //
AutoFdEntity::AutoFdEntity(AutoFdEntity& other) : pFdEntity(NULL) AutoFdEntity::AutoFdEntity(AutoFdEntity& other) : pFdEntity(NULL), pseudo_fd(-1)
{ {
S3FS_PRN_WARN("This method should not be called. Please check the caller."); S3FS_PRN_WARN("This method should not be called. Please check the caller.");
if(other.pFdEntity){ if(other.pFdEntity){
other.pFdEntity->Dup(); if(-1 != (pseudo_fd = other.pFdEntity->Dup(other.pseudo_fd))){
pFdEntity = other.pFdEntity; pFdEntity = other.pFdEntity;
}else{
S3FS_PRN_ERR("Failed duplicating fd in AutoFdEntity.");
}
} }
} }
@ -56,11 +59,12 @@ AutoFdEntity::~AutoFdEntity()
bool AutoFdEntity::Close() bool AutoFdEntity::Close()
{ {
if(pFdEntity){ if(pFdEntity){
if(!FdManager::get()->Close(pFdEntity)){ if(!FdManager::get()->Close(pFdEntity, pseudo_fd)){
S3FS_PRN_ERR("Failed to close fdentity."); S3FS_PRN_ERR("Failed to close fdentity.");
return false; return false;
} }
pFdEntity = NULL; pFdEntity = NULL;
pseudo_fd = -1;
} }
return true; return true;
} }
@ -69,48 +73,61 @@ bool AutoFdEntity::Close()
// This method touches the internal fdentity with. // This method touches the internal fdentity with.
// This is used to keep the file open. // This is used to keep the file open.
// //
bool AutoFdEntity::Detach() int AutoFdEntity::Detach()
{ {
if(!pFdEntity){ if(!pFdEntity){
S3FS_PRN_ERR("Does not have a associated FdEntity."); S3FS_PRN_ERR("Does not have a associated FdEntity.");
return -1;
}
int fd = pseudo_fd;
pseudo_fd = -1;
pFdEntity = NULL;
return fd;
}
bool AutoFdEntity::Attach(const char* path, int existfd)
{
Close();
if(NULL == (pFdEntity = FdManager::get()->GetFdEntity(path, existfd, false))){
S3FS_PRN_DBG("Could not find fd entity object(file=%s, existfd=%d)", path, existfd);
return false; return false;
} }
pFdEntity = NULL; pseudo_fd = existfd;
return true; return true;
} }
FdEntity* AutoFdEntity::Open(const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, bool no_fd_lock_wait)
{
Close();
if(NULL == (pFdEntity = FdManager::get()->Open(pseudo_fd, path, pmeta, size, time, flags, force_tmpfile, is_create, no_fd_lock_wait))){
pseudo_fd = -1;
return NULL;
}
return pFdEntity;
}
// [NOTE] // [NOTE]
// This method calls the FdManager method without incrementing the // the fd obtained by this method is not a newly created pseudo fd.
// reference count.
// This means that it will only be used to map to a file descriptor
// that was already open.
// //
FdEntity* AutoFdEntity::GetFdEntity(const char* path, int existfd, bool increase_ref) FdEntity* AutoFdEntity::GetExistFdEntiy(const char* path, int existfd)
{ {
Close(); Close();
if(NULL == (pFdEntity = FdManager::get()->GetFdEntity(path, existfd, increase_ref))){ FdEntity* ent;
S3FS_PRN_DBG("Could not find fd(file=%s, existfd=%d)", path, existfd); if(NULL == (ent = FdManager::get()->GetExistFdEntiy(path, existfd))){
return NULL; return NULL;
} }
return pFdEntity; return ent;
} }
FdEntity* AutoFdEntity::Open(const char* path, headers_t* pmeta, off_t size, time_t time, bool force_tmpfile, bool is_create, bool no_fd_lock_wait) FdEntity* AutoFdEntity::OpenExistFdEntiy(const char* path, int flags)
{ {
Close(); Close();
if(NULL == (pFdEntity = FdManager::get()->Open(path, pmeta, size, time, force_tmpfile, is_create, no_fd_lock_wait))){ if(NULL == (pFdEntity = FdManager::get()->OpenExistFdEntiy(path, pseudo_fd, flags))){
return NULL;
}
return pFdEntity;
}
FdEntity* AutoFdEntity::ExistOpen(const char* path, int existfd, bool ignore_existfd)
{
Close();
if(NULL == (pFdEntity = FdManager::get()->ExistOpen(path, existfd, ignore_existfd))){
return NULL; return NULL;
} }
return pFdEntity; return pFdEntity;
@ -128,8 +145,12 @@ bool AutoFdEntity::operator=(AutoFdEntity& other)
Close(); Close();
if(other.pFdEntity){ if(other.pFdEntity){
other.pFdEntity->Dup(); if(-1 != (pseudo_fd = other.pFdEntity->Dup(other.pseudo_fd))){
pFdEntity = other.pFdEntity; pFdEntity = other.pFdEntity;
}else{
S3FS_PRN_ERR("Failed duplicating fd in AutoFdEntity.");
return false;
}
} }
return true; return true;
} }

12
src/fdcache_auto.h
View File

@ -35,6 +35,7 @@ class AutoFdEntity
{ {
private: private:
FdEntity* pFdEntity; FdEntity* pFdEntity;
int pseudo_fd;
private: private:
AutoFdEntity(AutoFdEntity& other); AutoFdEntity(AutoFdEntity& other);
@ -45,10 +46,13 @@ class AutoFdEntity
~AutoFdEntity(); ~AutoFdEntity();
bool Close(); bool Close();
bool Detach(); int Detach();
FdEntity* GetFdEntity(const char* path, int existfd = -1, bool increase_ref = true); bool Attach(const char* path, int existfd);
FdEntity* Open(const char* path, headers_t* pmeta = NULL, off_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false); int GetPseudoFd() const { return pseudo_fd; }
FdEntity* ExistOpen(const char* path, int existfd = -1, bool ignore_existfd = false);
FdEntity* Open(const char* path, headers_t* pmeta = NULL, off_t size = -1, time_t time = -1, int flags = O_RDONLY, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false);
FdEntity* GetExistFdEntiy(const char* path, int existfd = -1);
FdEntity* OpenExistFdEntiy(const char* path, int flags = O_RDONLY);
}; };
#endif // S3FS_FDCACHE_AUTO_H_ #endif // S3FS_FDCACHE_AUTO_H_

474
src/fdcache_entity.cpp
View File

@ -94,8 +94,8 @@ ino_t FdEntity::GetInode(int fd)
// FdEntity methods // FdEntity methods
//------------------------------------------------ //------------------------------------------------
FdEntity::FdEntity(const char* tpath, const char* cpath) : FdEntity::FdEntity(const char* tpath, const char* cpath) :
is_lock_init(false), refcnt(0), path(SAFESTRPTR(tpath)), is_lock_init(false), path(SAFESTRPTR(tpath)),
fd(-1), pfile(NULL), inode(0), size_orgmeta(0), mp_start(0), mp_size(0), physical_fd(-1), pfile(NULL), inode(0), size_orgmeta(0),
cachepath(SAFESTRPTR(cpath)), is_meta_pending(false) cachepath(SAFESTRPTR(cpath)), is_meta_pending(false)
{ {
holding_mtime.tv_sec = -1; holding_mtime.tv_sec = -1;
@ -140,7 +140,15 @@ void FdEntity::Clear()
AutoLock auto_lock(&fdent_lock); AutoLock auto_lock(&fdent_lock);
AutoLock auto_data_lock(&fdent_data_lock); AutoLock auto_data_lock(&fdent_data_lock);
if(-1 != fd){ for(fdinfo_map_t::iterator iter = pseudo_fd_map.begin(); iter != pseudo_fd_map.end(); ++iter){
PseudoFdInfo* ppseudofdinfo = iter->second;
if(ppseudofdinfo){
delete ppseudofdinfo;
}
}
pseudo_fd_map.clear();
if(-1 != physical_fd){
if(!cachepath.empty()){ if(!cachepath.empty()){
// [NOTE] // [NOTE]
// Compare the inode of the existing cache file with the inode of // Compare the inode of the existing cache file with the inode of
@ -159,7 +167,7 @@ void FdEntity::Clear()
fclose(pfile); fclose(pfile);
pfile = NULL; pfile = NULL;
} }
fd = -1; physical_fd = -1;
inode = 0; inode = 0;
if(!mirrorpath.empty()){ if(!mirrorpath.empty()){
@ -170,7 +178,6 @@ void FdEntity::Clear()
} }
} }
pagelist.Init(0, false, false); pagelist.Init(0, false, false);
refcnt = 0;
path = ""; path = "";
cachepath = ""; cachepath = "";
} }
@ -195,20 +202,24 @@ ino_t FdEntity::GetInode()
return st.st_ino; return st.st_ino;
} }
void FdEntity::Close() void FdEntity::Close(int fd)
{ {
AutoLock auto_lock(&fdent_lock); AutoLock auto_lock(&fdent_lock);
S3FS_PRN_DBG("[path=%s][fd=%d][refcnt=%d]", path.c_str(), fd, (-1 != fd ? refcnt - 1 : refcnt)); S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d]", path.c_str(), fd, physical_fd);
if(-1 != fd){ // search pseudo fd and close it.
if(0 < refcnt){ fdinfo_map_t::iterator iter = pseudo_fd_map.find(fd);
refcnt--; if(pseudo_fd_map.end() != iter){
PseudoFdInfo* ppseudoinfo = iter->second;
pseudo_fd_map.erase(iter);
delete ppseudoinfo;
}else{ }else{
S3FS_PRN_EXIT("reference count underflow"); S3FS_PRN_WARN("Not found pseudo_fd(%d) in entity object(%s)", fd, path.c_str());
abort();
} }
if(0 == refcnt){
// check pseudo fd count
if(-1 != physical_fd && 0 == GetOpenCount(true)){
AutoLock auto_data_lock(&fdent_data_lock); AutoLock auto_data_lock(&fdent_data_lock);
if(!cachepath.empty()){ if(!cachepath.empty()){
// [NOTE] // [NOTE]
@ -228,7 +239,7 @@ void FdEntity::Close()
fclose(pfile); fclose(pfile);
pfile = NULL; pfile = NULL;
} }
fd = -1; physical_fd = -1;
inode = 0; inode = 0;
if(!mirrorpath.empty()){ if(!mirrorpath.empty()){
@ -238,19 +249,51 @@ void FdEntity::Close()
mirrorpath.erase(); mirrorpath.erase();
} }
} }
}
} }
int FdEntity::Dup(bool lock_already_held) int FdEntity::Dup(int fd, bool lock_already_held)
{ {
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
S3FS_PRN_DBG("[path=%s][fd=%d][refcnt=%d]", path.c_str(), fd, (-1 != fd ? refcnt + 1 : refcnt)); S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][pseudo fd count=%zu]", path.c_str(), fd, physical_fd, pseudo_fd_map.size());
if(-1 != fd){ if(-1 == physical_fd){
refcnt++; return -1;
} }
return fd; fdinfo_map_t::iterator iter = pseudo_fd_map.find(fd);
if(pseudo_fd_map.end() == iter){
S3FS_PRN_ERR("Not found pseudo_fd(%d) in entity object(%s) for physical_fd(%d)", fd, path.c_str(), physical_fd);
return -1;
}
PseudoFdInfo* org_pseudoinfo = iter->second;
PseudoFdInfo* ppseudoinfo = new PseudoFdInfo(physical_fd, (org_pseudoinfo ? org_pseudoinfo->GetFlags() : 0));
int pseudo_fd = ppseudoinfo->GetPseudoFd();
pseudo_fd_map[pseudo_fd] = ppseudoinfo;
return pseudo_fd;
}
int FdEntity::OpenPseudoFd(int flags, bool lock_already_held)
{
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
S3FS_PRN_DBG("[path=%s][physical_fd=%d][pseudo fd count=%zu]", path.c_str(), physical_fd, pseudo_fd_map.size());
if(-1 == physical_fd){
return -1;
}
PseudoFdInfo* ppseudoinfo = new PseudoFdInfo(physical_fd, flags);
int pseudo_fd = ppseudoinfo->GetPseudoFd();
pseudo_fd_map[pseudo_fd] = ppseudoinfo;
return pseudo_fd;
}
int FdEntity::GetOpenCount(bool lock_already_held)
{
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
return static_cast<int>(pseudo_fd_map.size());
} }
// //
@ -310,11 +353,64 @@ int FdEntity::OpenMirrorFile()
return mirrorfd; return mirrorfd;
} }
int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type type) bool FdEntity::FindPseudoFd(int fd, bool lock_already_held)
{
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
if(-1 == fd){
return false;
}
if(pseudo_fd_map.end() == pseudo_fd_map.find(fd)){
return false;
}
return true;
}
PseudoFdInfo* FdEntity::CheckPseudoFdFlags(int fd, bool writable, bool lock_already_held)
{
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
if(-1 == fd){
return NULL;
}
fdinfo_map_t::iterator iter = pseudo_fd_map.find(fd);
if(pseudo_fd_map.end() == iter || NULL == iter->second){
return NULL;
}
if(writable){
if(!iter->second->Writable()){
return NULL;
}
}else{
if(!iter->second->Readable()){
return NULL;
}
}
return iter->second;
}
bool FdEntity::IsUploading(bool lock_already_held)
{
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
for(fdinfo_map_t::const_iterator iter = pseudo_fd_map.begin(); iter != pseudo_fd_map.end(); ++iter){
PseudoFdInfo* ppseudoinfo = iter->second;
if(ppseudoinfo && ppseudoinfo->IsUploading()){
return true;
}
}
return false;
}
// [NOTE]
// If the open is successful, returns pseudo fd.
// If it fails, it returns an error code with a negative value.
//
int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, int flags, AutoLock::Type type)
{ {
AutoLock auto_lock(&fdent_lock, type); AutoLock auto_lock(&fdent_lock, type);
S3FS_PRN_DBG("[path=%s][fd=%d][size=%lld][time=%lld]", path.c_str(), fd, static_cast<long long>(size), static_cast<long long>(time)); S3FS_PRN_DBG("[path=%s][physical_fd=%d][size=%lld][time=%lld][flags=0x%x]", path.c_str(), physical_fd, static_cast<long long>(size), static_cast<long long>(time), flags);
if (!auto_lock.isLockAcquired()) { if (!auto_lock.isLockAcquired()) {
// had to wait for fd lock, return // had to wait for fd lock, return
@ -323,26 +419,22 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
} }
AutoLock auto_data_lock(&fdent_data_lock); AutoLock auto_data_lock(&fdent_data_lock);
if(-1 != fd){
// already opened, needs to increment refcnt. if(-1 != physical_fd){
Dup(/*lock_already_held=*/ true); //
// already open file
//
// check only file size(do not need to save cfs and time. // check only file size(do not need to save cfs and time.
if(0 <= size && pagelist.Size() != size){ if(0 <= size && pagelist.Size() != size){
// truncate temporary file size // truncate temporary file size
if(-1 == ftruncate(fd, size)){ if(-1 == ftruncate(physical_fd, size)){
S3FS_PRN_ERR("failed to truncate temporary file(%d) by errno(%d).", fd, errno); S3FS_PRN_ERR("failed to truncate temporary file(%d) by errno(%d).", physical_fd, errno);
if(0 < refcnt){
refcnt--;
}
return -errno; return -errno;
} }
// resize page list // resize page list
if(!pagelist.Resize(size, false, true)){ // Areas with increased size are modified if(!pagelist.Resize(size, false, true)){ // Areas with increased size are modified
S3FS_PRN_ERR("failed to truncate temporary file information(%d).", fd); S3FS_PRN_ERR("failed to truncate temporary file information(%d).", physical_fd);
if(0 < refcnt){
refcnt--;
}
return -EIO; return -EIO;
} }
} }
@ -355,15 +447,16 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
if(new_size < size_orgmeta){ if(new_size < size_orgmeta){
size_orgmeta = new_size; size_orgmeta = new_size;
} }
return 0;
}
}else{
//
// file is not opened yet
//
bool need_save_csf = false; // need to save(reset) cache stat file bool need_save_csf = false; // need to save(reset) cache stat file
bool is_truncate = false; // need to truncate bool is_truncate = false; // need to truncate
if(!cachepath.empty()){ if(!cachepath.empty()){
// using cache // using cache
struct stat st; struct stat st;
if(stat(cachepath.c_str(), &st) == 0){ if(stat(cachepath.c_str(), &st) == 0){
if(st.st_mtime < time){ if(st.st_mtime < time){
@ -378,15 +471,15 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
CacheFileStat cfstat(path.c_str()); CacheFileStat cfstat(path.c_str());
// try to open cache file // try to open cache file
if( -1 != (fd = open(cachepath.c_str(), O_RDWR)) && if( -1 != (physical_fd = open(cachepath.c_str(), O_RDWR)) &&
0 != (inode = FdEntity::GetInode(fd)) && 0 != (inode = FdEntity::GetInode(physical_fd)) &&
pagelist.Serialize(cfstat, false, inode) ) pagelist.Serialize(cfstat, false, inode) )
{ {
// succeed to open cache file and to load stats data // succeed to open cache file and to load stats data
memset(&st, 0, sizeof(struct stat)); memset(&st, 0, sizeof(struct stat));
if(-1 == fstat(fd, &st)){ if(-1 == fstat(physical_fd, &st)){
S3FS_PRN_ERR("fstat is failed. errno(%d)", errno); S3FS_PRN_ERR("fstat is failed. errno(%d)", errno);
fd = -1; physical_fd = -1;
inode = 0; inode = 0;
return (0 == errno ? -EIO : -errno); return (0 == errno ? -EIO : -errno);
} }
@ -408,13 +501,13 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
} }
}else{ }else{
if(-1 != fd){ if(-1 != physical_fd){
close(fd); close(physical_fd);
} }
inode = 0; inode = 0;
// could not open cache file or could not load stats data, so initialize it. // could not open cache file or could not load stats data, so initialize it.
if(-1 == (fd = open(cachepath.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0600))){ if(-1 == (physical_fd = open(cachepath.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0600))){
S3FS_PRN_ERR("failed to open file(%s). errno(%d)", cachepath.c_str(), errno); S3FS_PRN_ERR("failed to open file(%s). errno(%d)", cachepath.c_str(), errno);
// remove cache stat file if it is existed // remove cache stat file if it is existed
@ -426,7 +519,7 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
return (0 == errno ? -EIO : -errno); return (0 == errno ? -EIO : -errno);
} }
need_save_csf = true; // need to update page info need_save_csf = true; // need to update page info
inode = FdEntity::GetInode(fd); inode = FdEntity::GetInode(physical_fd);
if(-1 == size){ if(-1 == size){
size = 0; size = 0;
pagelist.Init(0, false, false); pagelist.Init(0, false, false);
@ -452,14 +545,14 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
return (0 == mirrorfd ? -EIO : mirrorfd); return (0 == mirrorfd ? -EIO : mirrorfd);
} }
// switch fd // switch fd
close(fd); close(physical_fd);
fd = mirrorfd; physical_fd = mirrorfd;
// make file pointer(for being same tmpfile) // make file pointer(for being same tmpfile)
if(NULL == (pfile = fdopen(fd, "wb"))){ if(NULL == (pfile = fdopen(physical_fd, "wb"))){
S3FS_PRN_ERR("failed to get fileno(%s). errno(%d)", cachepath.c_str(), errno); S3FS_PRN_ERR("failed to get fileno(%s). errno(%d)", cachepath.c_str(), errno);
close(fd); close(physical_fd);
fd = -1; physical_fd = -1;
inode = 0; inode = 0;
return (0 == errno ? -EIO : -errno); return (0 == errno ? -EIO : -errno);
} }
@ -469,7 +562,7 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
inode = 0; inode = 0;
// open temporary file // open temporary file
if(NULL == (pfile = tmpfile()) || -1 ==(fd = fileno(pfile))){ if(NULL == (pfile = tmpfile()) || -1 ==(physical_fd = fileno(pfile))){
S3FS_PRN_ERR("failed to open tmp file. err(%d)", errno); S3FS_PRN_ERR("failed to open tmp file. err(%d)", errno);
if(pfile){ if(pfile){
fclose(pfile); fclose(pfile);
@ -496,11 +589,11 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
// truncate cache(tmp) file // truncate cache(tmp) file
if(is_truncate){ if(is_truncate){
if(0 != ftruncate(fd, size) || 0 != fsync(fd)){ if(0 != ftruncate(physical_fd, size) || 0 != fsync(physical_fd)){
S3FS_PRN_ERR("ftruncate(%s) or fsync returned err(%d)", cachepath.c_str(), errno); S3FS_PRN_ERR("ftruncate(%s) or fsync returned err(%d)", cachepath.c_str(), errno);
fclose(pfile); fclose(pfile);
pfile = NULL; pfile = NULL;
fd = -1; physical_fd = -1;
inode = 0; inode = 0;
return (0 == errno ? -EIO : -errno); return (0 == errno ? -EIO : -errno);
} }
@ -514,9 +607,6 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
} }
} }
// init internal data
refcnt = 1;
// set original headers and size in it. // set original headers and size in it.
if(pmeta){ if(pmeta){
orgmeta = *pmeta; orgmeta = *pmeta;
@ -533,31 +623,37 @@ int FdEntity::Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type typ
S3FS_PRN_ERR("failed to set mtime. errno(%d)", errno); S3FS_PRN_ERR("failed to set mtime. errno(%d)", errno);
fclose(pfile); fclose(pfile);
pfile = NULL; pfile = NULL;
fd = -1; physical_fd = -1;
inode = 0; inode = 0;
return (0 == errno ? -EIO : -errno); return (0 == errno ? -EIO : -errno);
} }
} }
return 0; }
// create new pseudo fd, and set it to map
PseudoFdInfo* ppseudoinfo = new PseudoFdInfo(physical_fd, flags);
int pseudo_fd = ppseudoinfo->GetPseudoFd();
pseudo_fd_map[pseudo_fd] = ppseudoinfo;
return pseudo_fd;
} }
// [NOTE] // [NOTE]
// This method is called from only nocopyapi functions. // This method is called for only nocopyapi functions.
// So we do not check disk space for this option mode, if there is no enough // So we do not check disk space for this option mode, if there is no enough
// disk space this method will be failed. // disk space this method will be failed.
// //
bool FdEntity::OpenAndLoadAll(headers_t* pmeta, off_t* size, bool force_load) bool FdEntity::LoadAll(int fd, headers_t* pmeta, off_t* size, bool force_load)
{ {
AutoLock auto_lock(&fdent_lock); AutoLock auto_lock(&fdent_lock);
int result;
S3FS_PRN_INFO3("[path=%s][fd=%d]", path.c_str(), fd); S3FS_PRN_INFO3("[path=%s][pseudo_fd=%d][physical_fd=%d]", path.c_str(), fd, physical_fd);
if(-1 == fd){ if(-1 == physical_fd || !FindPseudoFd(fd, true)){
if(0 != Open(pmeta, /*size=*/ -1, /*time=*/ -1, AutoLock::ALREADY_LOCKED)){ S3FS_PRN_ERR("pseudo_fd(%d) and physical_fd(%d) for path(%s) is not opened yet", fd, physical_fd, path.c_str());
return false; return false;
} }
}
AutoLock auto_data_lock(&fdent_data_lock); AutoLock auto_data_lock(&fdent_data_lock);
if(force_load){ if(force_load){
@ -566,6 +662,7 @@ bool FdEntity::OpenAndLoadAll(headers_t* pmeta, off_t* size, bool force_load)
// //
// TODO: possibly do background for delay loading // TODO: possibly do background for delay loading
// //
int result;
if(0 != (result = Load(/*start=*/ 0, /*size=*/ 0, /*lock_already_held=*/ true))){ if(0 != (result = Load(/*start=*/ 0, /*size=*/ 0, /*lock_already_held=*/ true))){
S3FS_PRN_ERR("could not download, result(%d)", result); S3FS_PRN_ERR("could not download, result(%d)", result);
return false; return false;
@ -635,12 +732,12 @@ bool FdEntity::IsModified() const
bool FdEntity::GetStats(struct stat& st, bool lock_already_held) bool FdEntity::GetStats(struct stat& st, bool lock_already_held)
{ {
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
if(-1 == fd){ if(-1 == physical_fd){
return false; return false;
} }
memset(&st, 0, sizeof(struct stat)); memset(&st, 0, sizeof(struct stat));
if(-1 == fstat(fd, &st)){ if(-1 == fstat(physical_fd, &st)){
S3FS_PRN_ERR("fstat failed. errno(%d)", errno); S3FS_PRN_ERR("fstat failed. errno(%d)", errno);
return false; return false;
} }
@ -651,7 +748,7 @@ int FdEntity::SetCtime(struct timespec time, bool lock_already_held)
{ {
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
S3FS_PRN_INFO3("[path=%s][fd=%d][time=%lld]", path.c_str(), fd, static_cast<long long>(time.tv_sec)); S3FS_PRN_INFO3("[path=%s][physical_fd=%d][time=%lld]", path.c_str(), physical_fd, static_cast<long long>(time.tv_sec));
if(-1 == time.tv_sec){ if(-1 == time.tv_sec){
return 0; return 0;
@ -664,7 +761,7 @@ int FdEntity::SetAtime(struct timespec time, bool lock_already_held)
{ {
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
S3FS_PRN_INFO3("[path=%s][fd=%d][time=%lld]", path.c_str(), fd, static_cast<long long>(time.tv_sec)); S3FS_PRN_INFO3("[path=%s][physical_fd=%d][time=%lld]", path.c_str(), physical_fd, static_cast<long long>(time.tv_sec));
if(-1 == time.tv_sec){ if(-1 == time.tv_sec){
return 0; return 0;
@ -680,19 +777,19 @@ int FdEntity::SetMCtime(struct timespec mtime, struct timespec ctime, bool lock_
{ {
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
S3FS_PRN_INFO3("[path=%s][fd=%d][mtime=%lld][ctime=%lld]", path.c_str(), fd, static_cast<long long>(mtime.tv_sec), static_cast<long long>(ctime.tv_sec)); S3FS_PRN_INFO3("[path=%s][physical_fd=%d][mtime=%lld][ctime=%lld]", path.c_str(), physical_fd, static_cast<long long>(mtime.tv_sec), static_cast<long long>(ctime.tv_sec));
if(mtime.tv_sec < 0 || ctime.tv_sec < 0){ if(mtime.tv_sec < 0 || ctime.tv_sec < 0){
return 0; return 0;
} }
if(-1 != fd){ if(-1 != physical_fd){
struct timeval tv[2]; struct timeval tv[2];
tv[0].tv_sec = mtime.tv_sec; tv[0].tv_sec = mtime.tv_sec;
tv[0].tv_usec = mtime.tv_nsec / 1000; tv[0].tv_usec = mtime.tv_nsec / 1000;
tv[1].tv_sec = ctime.tv_sec; tv[1].tv_sec = ctime.tv_sec;
tv[1].tv_usec = ctime.tv_nsec / 1000; tv[1].tv_usec = ctime.tv_nsec / 1000;
if(-1 == futimes(fd, tv)){ if(-1 == futimes(physical_fd, tv)){
S3FS_PRN_ERR("futimes failed. errno(%d)", errno); S3FS_PRN_ERR("futimes failed. errno(%d)", errno);
return -errno; return -errno;
} }
@ -785,13 +882,13 @@ bool FdEntity::ClearHoldingMtime(bool lock_already_held)
if(!GetStats(st, true)){ if(!GetStats(st, true)){
return false; return false;
} }
if(-1 != fd){ if(-1 != physical_fd){
struct timeval tv[2]; struct timeval tv[2];
tv[0].tv_sec = holding_mtime.tv_sec; tv[0].tv_sec = holding_mtime.tv_sec;
tv[0].tv_usec = holding_mtime.tv_nsec / 1000; tv[0].tv_usec = holding_mtime.tv_nsec / 1000;
tv[1].tv_sec = st.st_ctime; tv[1].tv_sec = st.st_ctime;
tv[1].tv_usec = 0; tv[1].tv_usec = 0;
if(-1 == futimes(fd, tv)){ if(-1 == futimes(physical_fd, tv)){
S3FS_PRN_ERR("futimes failed. errno(%d)", errno); S3FS_PRN_ERR("futimes failed. errno(%d)", errno);
return false; return false;
} }
@ -814,7 +911,7 @@ bool FdEntity::ClearHoldingMtime(bool lock_already_held)
bool FdEntity::GetSize(off_t& size) bool FdEntity::GetSize(off_t& size)
{ {
AutoLock auto_lock(&fdent_lock); AutoLock auto_lock(&fdent_lock);
if(-1 == fd){ if(-1 == physical_fd){
return false; return false;
} }
@ -875,9 +972,9 @@ bool FdEntity::SetContentType(const char* path)
bool FdEntity::SetAllStatus(bool is_loaded) bool FdEntity::SetAllStatus(bool is_loaded)
{ {
S3FS_PRN_INFO3("[path=%s][fd=%d][%s]", path.c_str(), fd, is_loaded ? "loaded" : "unloaded"); S3FS_PRN_INFO3("[path=%s][physical_fd=%d][%s]", path.c_str(), physical_fd, is_loaded ? "loaded" : "unloaded");
if(-1 == fd){ if(-1 == physical_fd){
return false; return false;
} }
// [NOTE] // [NOTE]
@ -889,7 +986,7 @@ bool FdEntity::SetAllStatus(bool is_loaded)
// get file size // get file size
struct stat st; struct stat st;
memset(&st, 0, sizeof(struct stat)); memset(&st, 0, sizeof(struct stat));
if(-1 == fstat(fd, &st)){ if(-1 == fstat(physical_fd, &st)){
S3FS_PRN_ERR("fstat is failed. errno(%d)", errno); S3FS_PRN_ERR("fstat is failed. errno(%d)", errno);
return false; return false;
} }
@ -903,9 +1000,9 @@ int FdEntity::Load(off_t start, off_t size, bool lock_already_held, bool is_modi
{ {
AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); AutoLock auto_lock(&fdent_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
S3FS_PRN_DBG("[path=%s][fd=%d][offset=%lld][size=%lld]", path.c_str(), fd, static_cast<long long int>(start), static_cast<long long int>(size)); S3FS_PRN_DBG("[path=%s][physical_fd=%d][offset=%lld][size=%lld]", path.c_str(), physical_fd, static_cast<long long int>(start), static_cast<long long int>(size));
if(-1 == fd){ if(-1 == physical_fd){
return -EBADF; return -EBADF;
} }
AutoLock auto_data_lock(&fdent_data_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE); AutoLock auto_data_lock(&fdent_data_lock, lock_already_held ? AutoLock::ALREADY_LOCKED : AutoLock::NONE);
@ -930,12 +1027,12 @@ int FdEntity::Load(off_t start, off_t size, bool lock_already_held, bool is_modi
// download // download
if(S3fsCurl::GetMultipartSize() <= need_load_size && !nomultipart){ if(S3fsCurl::GetMultipartSize() <= need_load_size && !nomultipart){
// parallel request // parallel request
result = S3fsCurl::ParallelGetObjectRequest(path.c_str(), fd, iter->offset, need_load_size); result = S3fsCurl::ParallelGetObjectRequest(path.c_str(), physical_fd, iter->offset, need_load_size);
}else{ }else{
// single request // single request
if(0 < need_load_size){ if(0 < need_load_size){
S3fsCurl s3fscurl; S3fsCurl s3fscurl;
result = s3fscurl.GetObjectRequest(path.c_str(), fd, iter->offset, need_load_size); result = s3fscurl.GetObjectRequest(path.c_str(), physical_fd, iter->offset, need_load_size);
}else{ }else{
result = 0; result = 0;
} }
@ -959,13 +1056,18 @@ int FdEntity::Load(off_t start, off_t size, bool lock_already_held, bool is_modi
// [NOTICE] // [NOTICE]
// Need to lock before calling this method. // Need to lock before calling this method.
// //
int FdEntity::NoCacheLoadAndPost(off_t start, off_t size) int FdEntity::NoCacheLoadAndPost(PseudoFdInfo* pseudo_obj, off_t start, off_t size)
{ {
int result = 0; int result = 0;
S3FS_PRN_INFO3("[path=%s][fd=%d][offset=%lld][size=%lld]", path.c_str(), fd, static_cast<long long int>(start), static_cast<long long int>(size)); S3FS_PRN_INFO3("[path=%s][physical_fd=%d][offset=%lld][size=%lld]", path.c_str(), physical_fd, static_cast<long long int>(start), static_cast<long long int>(size));
if(-1 == fd){ if(!pseudo_obj){
S3FS_PRN_ERR("Pseudo obejct is NULL.");
return -EIO;
}
if(-1 == physical_fd){
return -EBADF; return -EBADF;
} }
@ -1004,7 +1106,7 @@ int FdEntity::NoCacheLoadAndPost(off_t start, off_t size)
} }
// download each multipart size(default 10MB) in unit // download each multipart size(default 10MB) in unit
for(off_t oneread = 0, totalread = (iter->offset < start ? start : 0); totalread < static_cast<off_t>(iter->bytes); totalread += oneread){ for(off_t oneread = 0, totalread = (iter->offset < start ? start : 0); totalread < static_cast<off_t>(iter->bytes); totalread += oneread){
int upload_fd = fd; int upload_fd = physical_fd;
off_t offset = iter->offset + totalread; off_t offset = iter->offset + totalread;
oneread = std::min(static_cast<off_t>(iter->bytes) - totalread, S3fsCurl::GetMultipartSize()); oneread = std::min(static_cast<off_t>(iter->bytes) - totalread, S3fsCurl::GetMultipartSize());
@ -1074,7 +1176,7 @@ int FdEntity::NoCacheLoadAndPost(off_t start, off_t size)
// already loaded area // already loaded area
} }
// single area upload by multipart post // single area upload by multipart post
if(0 != (result = NoCacheMultipartPost(upload_fd, offset, oneread))){ if(0 != (result = NoCacheMultipartPost(pseudo_obj, upload_fd, offset, oneread))){
S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(%d).", static_cast<long long int>(offset), static_cast<long long int>(oneread), upload_fd); S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(%d).", static_cast<long long int>(offset), static_cast<long long int>(oneread), upload_fd);
break; break;
} }
@ -1107,8 +1209,8 @@ int FdEntity::NoCacheLoadAndPost(off_t start, off_t size)
pagelist.Compress(); pagelist.Compress();
// fd data do empty // fd data do empty
if(-1 == ftruncate(fd, 0)){ if(-1 == ftruncate(physical_fd, 0)){
S3FS_PRN_ERR("failed to truncate file(%d), but continue...", fd); S3FS_PRN_ERR("failed to truncate file(%d), but continue...", physical_fd);
} }
} }
@ -1122,18 +1224,28 @@ int FdEntity::NoCacheLoadAndPost(off_t start, off_t size)
// At no disk space for caching object. // At no disk space for caching object.
// This method is starting multipart uploading. // This method is starting multipart uploading.
// //
int FdEntity::NoCachePreMultipartPost() int FdEntity::NoCachePreMultipartPost(PseudoFdInfo* pseudo_obj)
{ {
if(!pseudo_obj){
S3FS_PRN_ERR("Internal error, pseudo fd object pointer is null.");
return -EIO;
}
// initialize multipart upload values // initialize multipart upload values
upload_id.erase(); pseudo_obj->ClearUploadInfo(true);
etaglist.clear();
S3fsCurl s3fscurl(true); S3fsCurl s3fscurl(true);
std::string upload_id;
int result; int result;
if(0 != (result = s3fscurl.PreMultipartPostRequest(path.c_str(), orgmeta, upload_id, false))){ if(0 != (result = s3fscurl.PreMultipartPostRequest(path.c_str(), orgmeta, upload_id, false))){
return result; return result;
} }
s3fscurl.DestroyCurlHandle(); s3fscurl.DestroyCurlHandle();
// reset upload_id
if(!pseudo_obj->InitialUploadInfo(upload_id)){
return -EIO;
}
return 0; return 0;
} }
@ -1141,27 +1253,48 @@ int FdEntity::NoCachePreMultipartPost()
// At no disk space for caching object. // At no disk space for caching object.
// This method is uploading one part of multipart. // This method is uploading one part of multipart.
// //
int FdEntity::NoCacheMultipartPost(int tgfd, off_t start, off_t size) int FdEntity::NoCacheMultipartPost(PseudoFdInfo* pseudo_obj, int tgfd, off_t start, off_t size)
{ {
if(-1 == tgfd || upload_id.empty()){ if(-1 == tgfd || !pseudo_obj || !pseudo_obj->IsUploading()){
S3FS_PRN_ERR("Need to initialize for multipart post."); S3FS_PRN_ERR("Need to initialize for multipart post.");
return -EIO; return -EIO;
} }
// get upload id
std::string upload_id("");
if(!pseudo_obj->GetUploadId(upload_id)){
return -EIO;
}
// append new part and get it's etag string pointer
int partnum = 0;
std::string* petag = NULL;
if(!pseudo_obj->AppendUploadPart(start, size, false, &partnum, &petag)){
return -EIO;
}
S3fsCurl s3fscurl(true); S3fsCurl s3fscurl(true);
return s3fscurl.MultipartUploadRequest(upload_id, path.c_str(), tgfd, start, size, etaglist); return s3fscurl.MultipartUploadRequest(upload_id, path.c_str(), tgfd, start, size, partnum, petag);
} }
// [NOTE] // [NOTE]
// At no disk space for caching object. // At no disk space for caching object.
// This method is finishing multipart uploading. // This method is finishing multipart uploading.
// //
int FdEntity::NoCacheCompleteMultipartPost() int FdEntity::NoCacheCompleteMultipartPost(PseudoFdInfo* pseudo_obj)
{ {
if(upload_id.empty() || etaglist.empty()){ etaglist_t etaglist;
if(!pseudo_obj || !pseudo_obj->IsUploading() || !pseudo_obj->GetEtaglist(etaglist)){
S3FS_PRN_ERR("There is no upload id or etag list."); S3FS_PRN_ERR("There is no upload id or etag list.");
return -EIO; return -EIO;
} }
// get upload id
std::string upload_id("");
if(!pseudo_obj->GetUploadId(upload_id)){
return -EIO;
}
S3fsCurl s3fscurl(true); S3fsCurl s3fscurl(true);
int result; int result;
if(0 != (result = s3fscurl.CompleteMultipartPostRequest(path.c_str(), upload_id, etaglist))){ if(0 != (result = s3fscurl.CompleteMultipartPostRequest(path.c_str(), upload_id, etaglist))){
@ -1169,33 +1302,42 @@ int FdEntity::NoCacheCompleteMultipartPost()
} }
s3fscurl.DestroyCurlHandle(); s3fscurl.DestroyCurlHandle();
// reset values // clear multipart upload info
upload_id.erase(); pseudo_obj->ClearUploadInfo(false);
etaglist.clear();
mp_start = 0;
mp_size = 0;
return 0; return 0;
} }
off_t FdEntity::BytesModified() { off_t FdEntity::BytesModified()
{
AutoLock auto_lock(&fdent_data_lock); AutoLock auto_lock(&fdent_data_lock);
return pagelist.BytesModified(); return pagelist.BytesModified();
} }
int FdEntity::RowFlush(const char* tpath, bool force_sync) int FdEntity::RowFlush(int fd, const char* tpath, bool force_sync)
{ {
int result = 0; S3FS_PRN_INFO3("[tpath=%s][path=%s][pseudo_fd=%d][physical_fd=%d]", SAFESTRPTR(tpath), path.c_str(), fd, physical_fd);
AutoLock auto_lock(&fdent_lock); if(-1 == physical_fd){
return -EBADF;
}
int result = 0;
std::string tmppath = path; std::string tmppath = path;
headers_t tmporgmeta = orgmeta; headers_t tmporgmeta = orgmeta;
S3FS_PRN_INFO3("[tpath=%s][path=%s][fd=%d]", SAFESTRPTR(tpath), tmppath.c_str(), fd); AutoLock auto_lock(&fdent_lock);
if(-1 == fd){ // check pseudo fd and its flag
fdinfo_map_t::iterator miter = pseudo_fd_map.find(fd);
if(pseudo_fd_map.end() == miter || NULL == miter->second){
return -EBADF; return -EBADF;
} }
if(!miter->second->Writable()){
// If the entity is opened read-only, it will end normally without updating.
return 0;
}
PseudoFdInfo* pseudo_obj = miter->second;
AutoLock auto_lock2(&fdent_data_lock); AutoLock auto_lock2(&fdent_data_lock);
if(!force_sync && !pagelist.IsModified()){ if(!force_sync && !pagelist.IsModified()){
@ -1206,7 +1348,7 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
// If there is no loading all of the area, loading all area. // If there is no loading all of the area, loading all area.
off_t restsize = pagelist.GetTotalUnloadedPageSize(); off_t restsize = pagelist.GetTotalUnloadedPageSize();
if(0 < restsize){ if(0 < restsize){
if(0 == upload_id.length()){ if(!pseudo_obj->IsUploading()){
// check disk space // check disk space
if(ReserveDiskSpace(restsize)){ if(ReserveDiskSpace(restsize)){
// enough disk space // enough disk space
@ -1225,24 +1367,22 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
S3FS_PRN_WARN("Not enough local storage to flush: [path=%s][fd=%d]", path.c_str(), fd); S3FS_PRN_WARN("Not enough local storage to flush: [path=%s][fd=%d]", path.c_str(), fd);
return -ENOSPC; // No space left on device return -ENOSPC; // No space left on device
} }
if(0 != (result = NoCachePreMultipartPost())){ if(0 != (result = NoCachePreMultipartPost(pseudo_obj))){
S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result); S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result);
return result; return result;
} }
// upload all by multipart uploading // upload all by multipart uploading
if(0 != (result = NoCacheLoadAndPost())){ if(0 != (result = NoCacheLoadAndPost(pseudo_obj))){
S3FS_PRN_ERR("failed to upload all area by multipart uploading(errno=%d)", result); S3FS_PRN_ERR("failed to upload all area by multipart uploading(errno=%d)", result);
return result; return result;
} }
mp_start = 0;
mp_size = 0;
} }
}else{ }else{
// already start multipart uploading // already start multipart uploading
} }
} }
if(0 == upload_id.length()){ if(!pseudo_obj->IsUploading()){
// normal uploading // normal uploading
// //
// Make decision to do multi upload (or not) based upon file size // Make decision to do multi upload (or not) based upon file size
@ -1265,14 +1405,14 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
} }
// seek to head of file. // seek to head of file.
if(0 != lseek(fd, 0, SEEK_SET)){ if(0 != lseek(physical_fd, 0, SEEK_SET)){
S3FS_PRN_ERR("lseek error(%d)", errno); S3FS_PRN_ERR("lseek error(%d)", errno);
return -errno; return -errno;
} }
// backup upload file size // backup upload file size
struct stat st; struct stat st;
memset(&st, 0, sizeof(struct stat)); memset(&st, 0, sizeof(struct stat));
if(-1 == fstat(fd, &st)){ if(-1 == fstat(physical_fd, &st)){
S3FS_PRN_ERR("fstat is failed by errno(%d), but continue...", errno); S3FS_PRN_ERR("fstat is failed by errno(%d), but continue...", errno);
} }
@ -1299,11 +1439,11 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
} }
// multipart uploading with copy api // multipart uploading with copy api
result = S3fsCurl::ParallelMixMultipartUploadRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, fd, mixuppages); result = S3fsCurl::ParallelMixMultipartUploadRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd, mixuppages);
}else{ }else{
// multipart uploading not using copy api // multipart uploading not using copy api
result = S3fsCurl::ParallelMultipartUploadRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, fd); result = S3fsCurl::ParallelMultipartUploadRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd);
} }
}else{ }else{
// If there are unloaded pages, they are loaded at here. // If there are unloaded pages, they are loaded at here.
@ -1313,11 +1453,11 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
} }
S3fsCurl s3fscurl(true); S3fsCurl s3fscurl(true);
result = s3fscurl.PutRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, fd); result = s3fscurl.PutRequest(tpath ? tpath : tmppath.c_str(), tmporgmeta, physical_fd);
} }
// seek to head of file. // seek to head of file.
if(0 == result && 0 != lseek(fd, 0, SEEK_SET)){ if(0 == result && 0 != lseek(physical_fd, 0, SEEK_SET)){
S3FS_PRN_ERR("lseek error(%d)", errno); S3FS_PRN_ERR("lseek error(%d)", errno);
return -errno; return -errno;
} }
@ -1327,25 +1467,27 @@ int FdEntity::RowFlush(const char* tpath, bool force_sync)
}else{ }else{
// upload rest data // upload rest data
if(0 < mp_size){ off_t untreated_start = 0;
if(0 != (result = NoCacheMultipartPost(fd, mp_start, mp_size))){ off_t untreated_size = 0;
S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(%d).", static_cast<long long int>(mp_start), static_cast<long long int>(mp_size), fd); pseudo_obj->GetUntreated(untreated_start, untreated_size);
if(0 < untreated_size){
if(0 != (result = NoCacheMultipartPost(pseudo_obj, physical_fd, untreated_start, untreated_size))){
S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(%d).", static_cast<long long int>(untreated_start), static_cast<long long int>(untreated_size), physical_fd);
return result; return result;
} }
mp_start = 0; pseudo_obj->ClearUntreated();
mp_size = 0;
} }
// complete multipart uploading. // complete multipart uploading.
if(0 != (result = NoCacheCompleteMultipartPost())){ if(0 != (result = NoCacheCompleteMultipartPost(pseudo_obj))){
S3FS_PRN_ERR("failed to complete(finish) multipart post for file(%d).", fd); S3FS_PRN_ERR("failed to complete(finish) multipart post for file(%d).", physical_fd);
return result; return result;
} }
// truncate file to zero // truncate file to zero
if(-1 == ftruncate(fd, 0)){ if(-1 == ftruncate(physical_fd, 0)){
// So the file has already been removed, skip error. // So the file has already been removed, skip error.
S3FS_PRN_ERR("failed to truncate file(%d) to zero, but continue...", fd); S3FS_PRN_ERR("failed to truncate file(%d) to zero, but continue...", physical_fd);
} }
// put pading headers // put pading headers
if(0 != (result = UploadPendingMeta())){ if(0 != (result = UploadPendingMeta())){
return result; return result;
@ -1369,8 +1511,8 @@ bool FdEntity::ReserveDiskSpace(off_t size)
if(!pagelist.IsModified()){ if(!pagelist.IsModified()){
// try to clear all cache for this fd. // try to clear all cache for this fd.
pagelist.Init(pagelist.Size(), false, false); pagelist.Init(pagelist.Size(), false, false);
if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, pagelist.Size())){ if(-1 == ftruncate(physical_fd, 0) || -1 == ftruncate(physical_fd, pagelist.Size())){
S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd); S3FS_PRN_ERR("failed to truncate temporary file(%d).", physical_fd);
return false; return false;
} }
@ -1384,13 +1526,15 @@ bool FdEntity::ReserveDiskSpace(off_t size)
return FdManager::ReserveDiskSpace(size); return FdManager::ReserveDiskSpace(size);
} }
ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load) ssize_t FdEntity::Read(int fd, char* bytes, off_t start, size_t size, bool force_load)
{ {
S3FS_PRN_DBG("[path=%s][fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, static_cast<long long int>(start), size); S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, physical_fd, static_cast<long long int>(start), size);
if(-1 == fd){ if(-1 == physical_fd || NULL == CheckPseudoFdFlags(fd, false)){
S3FS_PRN_DBG("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not readable", fd, physical_fd, path.c_str());
return -EBADF; return -EBADF;
} }
AutoLock auto_lock(&fdent_lock); AutoLock auto_lock(&fdent_lock);
AutoLock auto_lock2(&fdent_data_lock); AutoLock auto_lock2(&fdent_data_lock);
@ -1438,20 +1582,23 @@ ssize_t FdEntity::Read(char* bytes, off_t start, size_t size, bool force_load)
} }
// Reading // Reading
if(-1 == (rsize = pread(fd, bytes, size, start))){ if(-1 == (rsize = pread(physical_fd, bytes, size, start))){
S3FS_PRN_ERR("pread failed. errno(%d)", errno); S3FS_PRN_ERR("pread failed. errno(%d)", errno);
return -errno; return -errno;
} }
return rsize; return rsize;
} }
ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size) ssize_t FdEntity::Write(int fd, const char* bytes, off_t start, size_t size)
{ {
S3FS_PRN_DBG("[path=%s][fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, static_cast<long long int>(start), size); S3FS_PRN_DBG("[path=%s][pseudo_fd=%d][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, physical_fd, static_cast<long long int>(start), size);
if(-1 == fd){ PseudoFdInfo* pseudo_obj = NULL;
if(-1 == physical_fd || NULL == (pseudo_obj = CheckPseudoFdFlags(fd, true))){
S3FS_PRN_DBG("pseudo_fd(%d) to physical_fd(%d) for path(%s) is not opened or not writable", fd, physical_fd, path.c_str());
return -EBADF; return -EBADF;
} }
// check if not enough disk space left BEFORE locking fd // check if not enough disk space left BEFORE locking fd
if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){ if(FdManager::IsCacheDir() && !FdManager::IsSafeDiskSpace(NULL, size)){
FdManager::get()->CleanupCacheDir(); FdManager::get()->CleanupCacheDir();
@ -1462,8 +1609,8 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
// check file size // check file size
if(pagelist.Size() < start){ if(pagelist.Size() < start){
// grow file size // grow file size
if(-1 == ftruncate(fd, start)){ if(-1 == ftruncate(physical_fd, start)){
S3FS_PRN_ERR("failed to truncate temporary file(%d).", fd); S3FS_PRN_ERR("failed to truncate temporary file(%d).", physical_fd);
return -errno; return -errno;
} }
// add new area // add new area
@ -1473,7 +1620,7 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
int result = 0; int result = 0;
ssize_t wsize; ssize_t wsize;
if(0 == upload_id.length()){ if(!pseudo_obj->IsUploading()){
// check disk space // check disk space
off_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size; off_t restsize = pagelist.GetTotalUnloadedPageSize(0, start) + size;
if(ReserveDiskSpace(restsize)){ if(ReserveDiskSpace(restsize)){
@ -1494,31 +1641,30 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
}else{ }else{
// no enough disk space // no enough disk space
if (nomultipart) { if (nomultipart) {
S3FS_PRN_WARN("Not enough local storage to cache write request: [path=%s][fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, static_cast<long long int>(start), size); S3FS_PRN_WARN("Not enough local storage to cache write request: [path=%s][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), physical_fd, static_cast<long long int>(start), size);
return -ENOSPC; // No space left on device return -ENOSPC; // No space left on device
} }
if ((start + static_cast<off_t>(size)) <= S3fsCurl::GetMultipartSize()) { if ((start + static_cast<off_t>(size)) <= S3fsCurl::GetMultipartSize()) {
S3FS_PRN_WARN("Not enough local storage to cache write request till multipart upload can start: [path=%s][fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, static_cast<long long int>(start), size); S3FS_PRN_WARN("Not enough local storage to cache write request till multipart upload can start: [path=%s][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), physical_fd, static_cast<long long int>(start), size);
return -ENOSPC; // No space left on device return -ENOSPC; // No space left on device
} }
if(0 != (result = NoCachePreMultipartPost())){ if(0 != (result = NoCachePreMultipartPost(pseudo_obj))){
S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result); S3FS_PRN_ERR("failed to switch multipart uploading with no cache(errno=%d)", result);
return result; return result;
} }
// start multipart uploading // start multipart uploading
if(0 != (result = NoCacheLoadAndPost(0, start))){ if(0 != (result = NoCacheLoadAndPost(pseudo_obj, 0, start))){
S3FS_PRN_ERR("failed to load uninitialized area and multipart uploading it(errno=%d)", result); S3FS_PRN_ERR("failed to load uninitialized area and multipart uploading it(errno=%d)", result);
return result; return result;
} }
mp_start = start; pseudo_obj->SetUntreated(start, 0);
mp_size = 0;
} }
}else{ }else{
// already start multipart uploading // already start multipart uploading
} }
// Writing // Writing
if(-1 == (wsize = pwrite(fd, bytes, size, start))){ if(-1 == (wsize = pwrite(physical_fd, bytes, size, start))){
S3FS_PRN_ERR("pwrite failed. errno(%d)", errno); S3FS_PRN_ERR("pwrite failed. errno(%d)", errno);
return -errno; return -errno;
} }
@ -1538,24 +1684,28 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
} }
// check multipart uploading // check multipart uploading
if(0 < upload_id.length()){ if(pseudo_obj->IsUploading()){
mp_size += wsize; off_t untreated_start = 0;
if(S3fsCurl::GetMultipartSize() <= mp_size){ off_t untreated_size = 0;
pseudo_obj->GetUntreated(untreated_start, untreated_size);
untreated_size += wsize;
if(S3fsCurl::GetMultipartSize() <= untreated_size){
// over one multipart size // over one multipart size
if(0 != (result = NoCacheMultipartPost(fd, mp_start, mp_size))){ if(0 != (result = NoCacheMultipartPost(pseudo_obj, physical_fd, untreated_start, untreated_size))){
S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(%d).", static_cast<long long int>(mp_start), static_cast<long long int>(mp_size), fd); S3FS_PRN_ERR("failed to multipart post(start=%lld, size=%lld) for file(%d).", static_cast<long long int>(untreated_start), static_cast<long long int>(untreated_size), physical_fd);
return result; return result;
} }
// [NOTE] // [NOTE]
// truncate file to zero and set length to part offset + size // truncate file to zero and set length to part offset + size
// after this, file length is (offset + size), but file does not use any disk space. // after this, file length is (offset + size), but file does not use any disk space.
// //
if(-1 == ftruncate(fd, 0) || -1 == ftruncate(fd, (mp_start + mp_size))){ if(-1 == ftruncate(physical_fd, 0) || -1 == ftruncate(physical_fd, (untreated_start + untreated_size))){
S3FS_PRN_ERR("failed to truncate file(%d).", fd); S3FS_PRN_ERR("failed to truncate file(%d).", physical_fd);
return -errno; return -errno;
} }
mp_start += mp_size; pseudo_obj->SetUntreated(untreated_start + untreated_size, 0);
mp_size = 0;
} }
} }
return wsize; return wsize;
@ -1595,7 +1745,7 @@ bool FdEntity::MergeOrgMeta(headers_t& updatemeta)
if(0 <= atime.tv_sec){ if(0 <= atime.tv_sec){
SetAtime(atime, true); SetAtime(atime, true);
} }
is_meta_pending |= !upload_id.empty(); is_meta_pending |= IsUploading(true);
return is_meta_pending; return is_meta_pending;
} }
@ -1656,9 +1806,9 @@ static int fallocate(int /*fd*/, int /*mode*/, off_t /*offset*/, off_t /*len*/)
// //
bool FdEntity::PunchHole(off_t start, size_t size) bool FdEntity::PunchHole(off_t start, size_t size)
{ {
S3FS_PRN_DBG("[path=%s][fd=%d][offset=%lld][size=%zu]", path.c_str(), fd, static_cast<long long int>(start), size); S3FS_PRN_DBG("[path=%s][physical_fd=%d][offset=%lld][size=%zu]", path.c_str(), physical_fd, static_cast<long long int>(start), size);
if(-1 == fd){ if(-1 == physical_fd){
return false; return false;
} }
AutoLock auto_lock(&fdent_data_lock); AutoLock auto_lock(&fdent_data_lock);
@ -1676,7 +1826,7 @@ bool FdEntity::PunchHole(off_t start, size_t size)
// try to punch hole to file // try to punch hole to file
for(fdpage_list_t::const_iterator iter = nodata_pages.begin(); iter != nodata_pages.end(); ++iter){ for(fdpage_list_t::const_iterator iter = nodata_pages.begin(); iter != nodata_pages.end(); ++iter){
if(0 != fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, iter->offset, iter->bytes)){ if(0 != fallocate(physical_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, iter->offset, iter->bytes)){
if(ENOSYS == errno || EOPNOTSUPP == errno){ if(ENOSYS == errno || EOPNOTSUPP == errno){
S3FS_PRN_ERR("filed to fallocate for punching hole to file with errno(%d), it maybe the fallocate function is not implemented in this kernel, or the file system does not support FALLOC_FL_PUNCH_HOLE.", errno); S3FS_PRN_ERR("filed to fallocate for punching hole to file with errno(%d), it maybe the fallocate function is not implemented in this kernel, or the file system does not support FALLOC_FL_PUNCH_HOLE.", errno);
}else{ }else{

33
src/fdcache_entity.h
View File

@ -23,6 +23,7 @@
#include "autolock.h" #include "autolock.h"
#include "fdcache_page.h" #include "fdcache_page.h"
#include "fdcache_fdinfo.h"
#include "metaheader.h" #include "metaheader.h"
//------------------------------------------------ //------------------------------------------------
@ -35,9 +36,9 @@ class FdEntity
pthread_mutex_t fdent_lock; pthread_mutex_t fdent_lock;
bool is_lock_init; bool is_lock_init;
int refcnt; // reference count
std::string path; // object path std::string path; // object path
int fd; // file descriptor(tmp file or cache file) int physical_fd; // physical file(cache or temporary file) descriptor
fdinfo_map_t pseudo_fd_map; // pseudo file descriptor information map
FILE* pfile; // file pointer(tmp file or cache file) FILE* pfile; // file pointer(tmp file or cache file)
ino_t inode; // inode number for cache file ino_t inode; // inode number for cache file
headers_t orgmeta; // original headers at opening headers_t orgmeta; // original headers at opening
@ -62,6 +63,8 @@ class FdEntity
void Clear(); void Clear();
ino_t GetInode(); ino_t GetInode();
int OpenMirrorFile(); int OpenMirrorFile();
int NoCacheLoadAndPost(off_t start = 0, off_t size = 0); // size=0 means loading to end
bool CheckPseudoFdFlags(int fd, bool writable, bool lock_already_held = false);
bool SetAllStatus(bool is_loaded); // [NOTE] not locking bool SetAllStatus(bool is_loaded); // [NOTE] not locking
bool SetAllStatusUnloaded() { return SetAllStatus(false); } bool SetAllStatusUnloaded() { return SetAllStatus(false); }
int UploadPendingMeta(); int UploadPendingMeta();
@ -73,16 +76,17 @@ class FdEntity
explicit FdEntity(const char* tpath = NULL, const char* cpath = NULL); explicit FdEntity(const char* tpath = NULL, const char* cpath = NULL);
~FdEntity(); ~FdEntity();
void Close(); void Close(int fd);
bool IsOpen() const { return (-1 != fd); } bool IsOpen() const { return (-1 != physical_fd); }
int Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type type); bool FindPseudoFd(int fd, bool lock_already_held = false);
bool OpenAndLoadAll(headers_t* pmeta = NULL, off_t* size = NULL, bool force_load = false); int Open(headers_t* pmeta, off_t size, time_t time, int flags, AutoLock::Type type);
int Dup(bool lock_already_held = false); bool LoadAll(int fd, headers_t* pmeta = NULL, off_t* size = NULL, bool force_load = false);
int GetRefCnt() const { return refcnt; } // [NOTE] Use only debugging int Dup(int fd, bool lock_already_held = false);
int OpenPseudoFd(int flags = O_RDONLY, bool lock_already_held = false);
int GetOpenCount(bool lock_already_held = false);
const char* GetPath() const { return path.c_str(); } const char* GetPath() const { return path.c_str(); }
bool RenamePath(const std::string& newpath, std::string& fentmapkey); bool RenamePath(const std::string& newpath, std::string& fentmapkey);
int GetFd() const { return fd; } int GetPhysicalFd() const { return physical_fd; }
bool IsModified() const; bool IsModified() const;
bool MergeOrgMeta(headers_t& updatemeta); bool MergeOrgMeta(headers_t& updatemeta);
@ -105,17 +109,16 @@ class FdEntity
bool SetContentType(const char* path); bool SetContentType(const char* path);
int Load(off_t start = 0, off_t size = 0, bool lock_already_held = false, bool is_modified_flag = false); // size=0 means loading to end int Load(off_t start = 0, off_t size = 0, bool lock_already_held = false, bool is_modified_flag = false); // size=0 means loading to end
int NoCacheLoadAndPost(off_t start = 0, off_t size = 0); // size=0 means loading to end
int NoCachePreMultipartPost(); int NoCachePreMultipartPost();
int NoCacheMultipartPost(int tgfd, off_t start, off_t size); int NoCacheMultipartPost(int tgfd, off_t start, off_t size);
int NoCacheCompleteMultipartPost(); int NoCacheCompleteMultipartPost();
off_t BytesModified(); off_t BytesModified();
int RowFlush(const char* tpath, bool force_sync = false); int RowFlush(int fd, const char* tpath, bool force_sync = false);
int Flush(bool force_sync = false) { return RowFlush(NULL, force_sync); } int Flush(int fd, bool force_sync = false) { return RowFlush(fd, NULL, force_sync); }
ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false); ssize_t Read(int fd, char* bytes, off_t start, size_t size, bool force_load = false);
ssize_t Write(const char* bytes, off_t start, size_t size); ssize_t Write(int fd, const char* bytes, off_t start, size_t size);
bool ReserveDiskSpace(off_t size); bool ReserveDiskSpace(off_t size);
bool PunchHole(off_t start = 0, size_t size = 0); bool PunchHole(off_t start = 0, size_t size = 0);

98
src/fdcache_fdinfo.cpp Normal file
View File

@ -0,0 +1,98 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Takeshi Nakatani <ggtakec.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <cstdio>
#include <cstdlib>
#include <algorithm>
#include "common.h"
#include "s3fs.h"
#include "fdcache_fdinfo.h"
#include "fdcache_pseudofd.h"
#include "autolock.h"
//------------------------------------------------
// PseudoFdInfo methods
//------------------------------------------------
PseudoFdInfo::PseudoFdInfo(int fd, int open_flags) : pseudo_fd(-1), physical_fd(fd), flags(0) //, is_lock_init(false)
{
if(-1 != physical_fd){
pseudo_fd = PseudoFdManager::Get();
flags = open_flags;
}
}
PseudoFdInfo::~PseudoFdInfo()
{
Clear();
}
bool PseudoFdInfo::Clear()
{
if(-1 != pseudo_fd){
PseudoFdManager::Release(pseudo_fd);
}
pseudo_fd = -1;
physical_fd = -1;
return true;
}
bool PseudoFdInfo::Set(int fd, int open_flags)
{
if(-1 == fd){
return false;
}
Clear();
physical_fd = fd;
pseudo_fd = PseudoFdManager::Get();
flags = open_flags;
return true;
}
bool PseudoFdInfo::Writable() const
{
if(-1 == pseudo_fd){
return false;
}
if(0 == (flags & (O_WRONLY | O_RDWR))){
return false;
}
return true;
}
bool PseudoFdInfo::Readable() const
{
if(-1 == pseudo_fd){
return false;
}
// O_RDONLY is 0x00, it means any pattern is readable.
return true;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

61
src/fdcache_fdinfo.h Normal file
View File

@ -0,0 +1,61 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_FDCACHE_FDINFO_H_
#define S3FS_FDCACHE_FDINFO_H_
//------------------------------------------------
// Class PseudoFdInfo
//------------------------------------------------
class PseudoFdInfo
{
private:
int pseudo_fd;
int physical_fd;
int flags; // flags at open
private:
bool Clear();
public:
PseudoFdInfo(int fd = -1, int open_flags = 0);
~PseudoFdInfo();
int GetPhysicalFd() const { return physical_fd; }
int GetPseudoFd() const { return pseudo_fd; }
int GetFlags() const { return flags; }
bool Writable() const;
bool Readable() const;
bool Set(int fd, int open_flags);
};
typedef std::map<int, class PseudoFdInfo*> fdinfo_map_t;
#endif // S3FS_FDCACHE_FDINFO_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

134
src/fdcache_pseudofd.cpp Normal file
View File

@ -0,0 +1,134 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Takeshi Nakatani <ggtakec.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <cstdio>
#include <cstdlib>
#include <algorithm>
#include "common.h"
#include "s3fs.h"
#include "fdcache_pseudofd.h"
#include "autolock.h"
//------------------------------------------------
// Symbols
//------------------------------------------------
// [NOTE]
// The minimum pseudo fd value starts 2.
// This is to avoid mistakes for 0(stdout) and 1(stderr), which are usually used.
//
#define MIN_PSEUDOFD_NUMBER 2
//------------------------------------------------
// PseudoFdManager class methods
//------------------------------------------------
PseudoFdManager& PseudoFdManager::GetManager()
{
static PseudoFdManager singleton;
return singleton;
}
int PseudoFdManager::Get()
{
return (PseudoFdManager::GetManager()).CreatePseudoFd();
}
bool PseudoFdManager::Release(int fd)
{
return (PseudoFdManager::GetManager()).ReleasePseudoFd(fd);
}
//------------------------------------------------
// PseudoFdManager methods
//------------------------------------------------
PseudoFdManager::PseudoFdManager() : is_lock_init(false)
{
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
#if S3FS_PTHREAD_ERRORCHECK
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
#endif
int result;
if(0 != (result = pthread_mutex_init(&pseudofd_list_lock, &attr))){
S3FS_PRN_CRIT("failed to init pseudofd_list_lock: %d", result);
abort();
}
is_lock_init = true;
}
PseudoFdManager::~PseudoFdManager()
{
if(is_lock_init){
int result;
if(0 != (result = pthread_mutex_destroy(&pseudofd_list_lock))){
S3FS_PRN_CRIT("failed to destroy pseudofd_list_lock: %d", result);
abort();
}
is_lock_init = false;
}
}
int PseudoFdManager::GetUnusedMinPseudoFd() const
{
int min_fd = MIN_PSEUDOFD_NUMBER;
// Look for the first discontinuous value.
for(pseudofd_list_t::const_iterator iter = pseudofd_list.begin(); iter != pseudofd_list.end(); ++iter){
if(min_fd == (*iter)){
++min_fd;
}else if(min_fd < (*iter)){
break;
}
}
return min_fd;
}
int PseudoFdManager::CreatePseudoFd()
{
AutoLock auto_lock(&pseudofd_list_lock);
int new_fd = PseudoFdManager::GetUnusedMinPseudoFd();
pseudofd_list.push_back(new_fd);
std::sort(pseudofd_list.begin(), pseudofd_list.end());
return new_fd;
}
bool PseudoFdManager::ReleasePseudoFd(int fd)
{
AutoLock auto_lock(&pseudofd_list_lock);
for(pseudofd_list_t::iterator iter = pseudofd_list.begin(); iter != pseudofd_list.end(); ++iter){
if(fd == (*iter)){
pseudofd_list.erase(iter);
return true;
}
}
return false;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

65
src/fdcache_pseudofd.h Normal file
View File

@ -0,0 +1,65 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_FDCACHE_PSEUDOFD_H_
#define S3FS_FDCACHE_PSEUDOFD_H_
//------------------------------------------------
// Typdefs
//------------------------------------------------
// List of pseudo fd in use
//
typedef std::vector<int> pseudofd_list_t;
//------------------------------------------------
// Class PseudoFdManager
//------------------------------------------------
class PseudoFdManager
{
private:
pseudofd_list_t pseudofd_list;
bool is_lock_init;
pthread_mutex_t pseudofd_list_lock; // protects pseudofd_list
private:
static PseudoFdManager& GetManager();
PseudoFdManager();
~PseudoFdManager();
int GetUnusedMinPseudoFd() const;
int CreatePseudoFd();
bool ReleasePseudoFd(int fd);
public:
static int Get();
static bool Release(int fd);
};
#endif // S3FS_FDCACHE_PSEUDOFD_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

133
src/s3fs.cpp
View File

@ -119,7 +119,7 @@ static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t
static int check_object_access(const char* path, int mask, struct stat* pstbuf); static int check_object_access(const char* path, int mask, struct stat* pstbuf);
static int check_object_owner(const char* path, struct stat* pstbuf); static int check_object_owner(const char* path, struct stat* pstbuf);
static int check_parent_object_access(const char* path, int mask); static int check_parent_object_access(const char* path, int mask);
static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, bool is_load = false); static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, int flags = O_RDONLY, bool is_load = false);
static bool multi_head_callback(S3fsCurl* s3fscurl); static bool multi_head_callback(S3fsCurl* s3fscurl);
static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl); static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl);
static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf, fuse_fill_dir_t filler); static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf, fuse_fill_dir_t filler);
@ -700,7 +700,7 @@ bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& sse
return true; return true;
} }
static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, bool is_load) static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, int flags, bool is_load)
{ {
int result; int result;
struct stat stobj; struct stat stobj;
@ -717,12 +717,12 @@ static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char*
time_t mtime = (!S_ISREG(stobj.st_mode) && !S_ISLNK(stobj.st_mode)) ? -1 : stobj.st_mtime; time_t mtime = (!S_ISREG(stobj.st_mode) && !S_ISLNK(stobj.st_mode)) ? -1 : stobj.st_mtime;
bool force_tmpfile = S_ISREG(stobj.st_mode) ? false : true; bool force_tmpfile = S_ISREG(stobj.st_mode) ? false : true;
if(NULL == (ent = autoent.Open(path, &meta, stobj.st_size, mtime, force_tmpfile, true))){ if(NULL == (ent = autoent.Open(path, &meta, stobj.st_size, mtime, flags, force_tmpfile, true))){
S3FS_PRN_ERR("Could not open file. errno(%d)", errno); S3FS_PRN_ERR("Could not open file. errno(%d)", errno);
return -EIO; return -EIO;
} }
// load // load
if(is_load && !ent->OpenAndLoadAll(&meta)){ if(is_load && !ent->LoadAll(autoent.GetPseudoFd(), &meta)){
S3FS_PRN_ERR("Could not load file. errno(%d)", errno); S3FS_PRN_ERR("Could not load file. errno(%d)", errno);
autoent.Close(); autoent.Close();
return -EIO; return -EIO;
@ -780,7 +780,7 @@ static int s3fs_getattr(const char* _path, struct stat* stbuf)
if(stbuf){ if(stbuf){
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL != (ent = autoent.ExistOpen(path))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
struct stat tmpstbuf; struct stat tmpstbuf;
if(ent->GetStats(tmpstbuf)){ if(ent->GetStats(tmpstbuf)){
stbuf->st_size = tmpstbuf.st_size; stbuf->st_size = tmpstbuf.st_size;
@ -811,7 +811,7 @@ static int s3fs_readlink(const char* _path, char* buf, size_t size)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
int result; int result;
if(0 != (result = get_local_fent(autoent, &ent, path))){ if(0 != (result = get_local_fent(autoent, &ent, path, O_RDONLY))){
S3FS_PRN_ERR("could not get fent(file=%s)", path); S3FS_PRN_ERR("could not get fent(file=%s)", path);
return result; return result;
} }
@ -826,7 +826,7 @@ static int s3fs_readlink(const char* _path, char* buf, size_t size)
} }
// Read // Read
ssize_t ressize; ssize_t ressize;
if(0 > (ressize = ent->Read(buf, 0, readsize))){ if(0 > (ressize = ent->Read(autoent.GetPseudoFd(), buf, 0, readsize))){
S3FS_PRN_ERR("could not read file(file=%s, ressize=%zd)", path, ressize); S3FS_PRN_ERR("could not read file(file=%s, ressize=%zd)", path, ressize);
return static_cast<int>(ressize); return static_cast<int>(ressize);
} }
@ -994,13 +994,12 @@ static int s3fs_create(const char* _path, mode_t mode, struct fuse_file_info* fi
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.Open(path, &meta, 0, -1, false, true))){ if(NULL == (ent = autoent.Open(path, &meta, 0, -1, fi->flags, false, true))){
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
return -EIO; return -EIO;
} }
ent->MarkDirtyNewFile(); ent->MarkDirtyNewFile();
autoent.Detach(); // KEEP fdentity open fi->fh = autoent.Detach(); // KEEP fdentity open;
fi->fh = ent->GetFd();
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -1191,19 +1190,25 @@ static int s3fs_symlink(const char* _from, const char* _to)
{ // scope for AutoFdEntity { // scope for AutoFdEntity
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.Open(to, &headers, 0, -1, true, true))){ if(NULL == (ent = autoent.Open(to, &headers, 0, -1, O_RDWR, true, true))){
S3FS_PRN_ERR("could not open tmpfile(errno=%d)", errno); S3FS_PRN_ERR("could not open tmpfile(errno=%d)", errno);
return -errno; return -errno;
} }
// write(without space words) // write(without space words)
strFrom = trim(std::string(from)); strFrom = trim(std::string(from));
ssize_t from_size = static_cast<ssize_t>(strFrom.length()); ssize_t from_size = static_cast<ssize_t>(strFrom.length());
if(from_size != ent->Write(strFrom.c_str(), 0, from_size)){ ssize_t ressize;
S3FS_PRN_ERR("could not write tmpfile(errno=%d)", errno); if(from_size != (ressize = ent->Write(autoent.GetPseudoFd(), strFrom.c_str(), 0, from_size))){
return -errno; if(ressize < 0){
S3FS_PRN_ERR("could not write tmpfile(errno=%d)", static_cast<int>(ressize));
return static_cast<int>(ressize);
}else{
S3FS_PRN_ERR("could not write tmpfile %zd byte(errno=%d)", ressize, errno);
return (0 == errno ? -EIO : -errno);
}
} }
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_WARN("could not upload tmpfile(result=%d)", result); S3FS_PRN_WARN("could not upload tmpfile(result=%d)", result);
} }
} }
@ -1254,11 +1259,11 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
// update time // update time
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.ExistOpen(from, -1, !FdManager::IsCacheDir()))){ if(NULL == (ent = autoent.OpenExistFdEntiy(from))){
// no opened fd // no opened fd
if(FdManager::IsCacheDir()){ if(FdManager::IsCacheDir()){
// create cache file if be needed // create cache file if be needed
ent = autoent.Open(from, &meta, buf.st_size, -1, false, true); ent = autoent.Open(from, &meta, buf.st_size, -1, O_RDONLY, false, true);
} }
if(ent){ if(ent){
struct timespec mtime = get_mtime(meta); struct timespec mtime = get_mtime(meta);
@ -1317,7 +1322,7 @@ static int rename_object_nocopy(const char* from, const char* to, bool update_ct
{ // scope for AutoFdEntity { // scope for AutoFdEntity
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(0 != (result = get_local_fent(autoent, &ent, from, true))){ if(0 != (result = get_local_fent(autoent, &ent, from, O_RDWR, true))){
S3FS_PRN_ERR("could not open and read file(%s)", from); S3FS_PRN_ERR("could not open and read file(%s)", from);
return result; return result;
} }
@ -1335,7 +1340,7 @@ static int rename_object_nocopy(const char* from, const char* to, bool update_ct
} }
// upload // upload
if(0 != (result = ent->RowFlush(to, true))){ if(0 != (result = ent->RowFlush(autoent.GetPseudoFd(), to, true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result); S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result);
return result; return result;
} }
@ -1573,8 +1578,8 @@ static int s3fs_rename(const char* _from, const char* _to)
{ // scope for AutoFdEntity { // scope for AutoFdEntity
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL != (ent = autoent.ExistOpen(from))){ if(NULL != (ent = autoent.OpenExistFdEntiy(from, O_RDWR))){
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result); S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result);
return result; return result;
} }
@ -1673,7 +1678,7 @@ static int s3fs_chmod(const char* _path, mode_t mode)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
if(ent->MergeOrgMeta(updatemeta)){ if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading. // meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete. // then the meta is pending and accumulated to be put after the upload is complete.
@ -1755,7 +1760,7 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
// open & load // open & load
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), true))){ if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
return result; return result;
} }
@ -1767,7 +1772,7 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
ent->SetMode(mode); ent->SetMode(mode);
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
return result; return result;
} }
@ -1850,7 +1855,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
if(ent->MergeOrgMeta(updatemeta)){ if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading. // meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete. // then the meta is pending and accumulated to be put after the upload is complete.
@ -1939,7 +1944,7 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
// open & load // open & load
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), true))){ if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
return result; return result;
} }
@ -1952,7 +1957,7 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
ent->SetGId(gid); ent->SetGId(gid);
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
return result; return result;
} }
@ -2032,7 +2037,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2])
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
bool keep_mtime = false; bool keep_mtime = false;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
if(ent->MergeOrgMeta(updatemeta)){ if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading. // meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete. // then the meta is pending and accumulated to be put after the upload is complete.
@ -2131,7 +2136,7 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2])
// open & load // open & load
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), true))){ if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
return result; return result;
} }
@ -2149,7 +2154,7 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2])
} }
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
return result; return result;
} }
@ -2184,7 +2189,7 @@ static int s3fs_truncate(const char* _path, off_t size)
// Get file information // Get file information
if(0 == (result = get_object_attribute(path, NULL, &meta))){ if(0 == (result = get_object_attribute(path, NULL, &meta))){
// Exists -> Get file(with size) // Exists -> Get file(with size)
if(NULL == (ent = autoent.Open(path, &meta, size, -1, false, true))){ if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, false, true))){
S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno); S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno);
return -EIO; return -EIO;
} }
@ -2209,14 +2214,14 @@ static int s3fs_truncate(const char* _path, off_t size)
meta["x-amz-meta-uid"] = str(pcxt->uid); meta["x-amz-meta-uid"] = str(pcxt->uid);
meta["x-amz-meta-gid"] = str(pcxt->gid); meta["x-amz-meta-gid"] = str(pcxt->gid);
if(NULL == (ent = autoent.Open(path, &meta, size, -1, true, true))){ if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, true, true))){
S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno); S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno);
return -EIO; return -EIO;
} }
} }
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result); S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result);
return result; return result;
} }
@ -2247,11 +2252,7 @@ static int s3fs_open(const char* _path, struct fuse_file_info* fi)
// and only the Stats cache exists. // and only the Stats cache exists.
// //
if(StatCache::getStatCacheData()->HasStat(path)){ if(StatCache::getStatCacheData()->HasStat(path)){
AutoFdEntity autoent_local; if(!FdManager::HasOpenEntityFd(path)){
if(NULL == autoent_local.ExistOpen(path, -1, true)){
if((result = s3fs_flush(_path, fi)) != 0){
S3FS_PRN_ERR("could not flush(%s): result=%d", path, result);
}
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
} }
} }
@ -2284,7 +2285,7 @@ static int s3fs_open(const char* _path, struct fuse_file_info* fi)
FdEntity* ent; FdEntity* ent;
headers_t meta; headers_t meta;
get_object_attribute(path, NULL, &meta, true, NULL, true); // no truncate cache get_object_attribute(path, NULL, &meta, true, NULL, true); // no truncate cache
if(NULL == (ent = autoent.Open(path, &meta, st.st_size, st.st_mtime, false, true))){ if(NULL == (ent = autoent.Open(path, &meta, st.st_size, st.st_mtime, fi->flags, false, true))){
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
return -EIO; return -EIO;
} }
@ -2293,14 +2294,13 @@ static int s3fs_open(const char* _path, struct fuse_file_info* fi)
time_t now = time(NULL); time_t now = time(NULL);
struct timespec ts = {now, 0}; struct timespec ts = {now, 0};
ent->SetMCtime(ts, ts); ent->SetMCtime(ts, ts);
if(0 != (result = ent->RowFlush(path, true))){ if(0 != (result = ent->RowFlush(autoent.GetPseudoFd(), path, true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result); S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result);
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
return result; return result;
} }
} }
autoent.Detach(); // KEEP fdentity open fi->fh = autoent.Detach(); // KEEP fdentity open;
fi->fh = ent->GetFd();
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -2316,13 +2316,10 @@ static int s3fs_read(const char* _path, char* buf, size_t size, off_t offset, st
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.ExistOpen(path, static_cast<int>(fi->fh)))){ if(NULL == (ent = autoent.GetExistFdEntiy(path, static_cast<int>(fi->fh)))){
S3FS_PRN_ERR("could not find opened fd(%s)", path); S3FS_PRN_ERR("could not find opened fd(%s)", path);
return -EIO; return -EIO;
} }
if(ent->GetFd() != static_cast<int>(fi->fh)){
S3FS_PRN_WARN("different fd(%d - %llu)", ent->GetFd(), (unsigned long long)(fi->fh));
}
// check real file size // check real file size
off_t realsize = 0; off_t realsize = 0;
@ -2331,7 +2328,7 @@ static int s3fs_read(const char* _path, char* buf, size_t size, off_t offset, st
return 0; return 0;
} }
if(0 > (res = ent->Read(buf, offset, size, false))){ if(0 > (res = ent->Read(static_cast<int>(fi->fh), buf, offset, size, false))){
S3FS_PRN_WARN("failed to read file(%s). result=%zd", path, res); S3FS_PRN_WARN("failed to read file(%s). result=%zd", path, res);
} }
@ -2347,20 +2344,18 @@ static int s3fs_write(const char* _path, const char* buf, size_t size, off_t off
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.ExistOpen(path, static_cast<int>(fi->fh)))){ if(NULL == (ent = autoent.GetExistFdEntiy(path, static_cast<int>(fi->fh)))){
S3FS_PRN_ERR("could not find opened fd(%s)", path); S3FS_PRN_ERR("could not find opened fd(%s)", path);
return -EIO; return -EIO;
} }
if(ent->GetFd() != static_cast<int>(fi->fh)){
S3FS_PRN_WARN("different fd(%d - %llu)", ent->GetFd(), (unsigned long long)(fi->fh)); if(0 > (res = ent->Write(static_cast<int>(fi->fh), buf, offset, size))){
}
if(0 > (res = ent->Write(buf, offset, size))){
S3FS_PRN_WARN("failed to write file(%s). result=%zd", path, res); S3FS_PRN_WARN("failed to write file(%s). result=%zd", path, res);
} }
if(max_dirty_data != -1 && ent->BytesModified() >= max_dirty_data){ if(max_dirty_data != -1 && ent->BytesModified() >= max_dirty_data){
int flushres; int flushres;
if(0 != (flushres = ent->RowFlush(path, true))){ if(0 != (flushres = ent->RowFlush(autoent.GetPseudoFd(), path, true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", path, flushres); S3FS_PRN_ERR("could not upload file(%s): result=%d", path, flushres);
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
return flushres; return flushres;
@ -2408,10 +2403,10 @@ static int s3fs_flush(const char* _path, struct fuse_file_info* fi)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL != (ent = autoent.ExistOpen(path, static_cast<int>(fi->fh)))){ if(NULL != (ent = autoent.GetExistFdEntiy(path, static_cast<int>(fi->fh)))){
ent->UpdateMtime(true); // clear the flag not to update mtime. ent->UpdateMtime(true); // clear the flag not to update mtime.
ent->UpdateCtime(); ent->UpdateCtime();
result = ent->Flush(false); result = ent->Flush(static_cast<int>(fi->fh), false);
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
} }
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -2431,12 +2426,12 @@ static int s3fs_fsync(const char* _path, int datasync, struct fuse_file_info* fi
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL != (ent = autoent.ExistOpen(path, static_cast<int>(fi->fh)))){ if(NULL != (ent = autoent.GetExistFdEntiy(path, static_cast<int>(fi->fh)))){
if(0 == datasync){ if(0 == datasync){
ent->UpdateMtime(); ent->UpdateMtime();
ent->UpdateCtime(); ent->UpdateCtime();
} }
result = ent->Flush(false); result = ent->Flush(static_cast<int>(fi->fh), false);
} }
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -2468,29 +2463,21 @@ static int s3fs_release(const char* _path, struct fuse_file_info* fi)
{ // scope for AutoFdEntity { // scope for AutoFdEntity
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent;
// [NOTE] // [NOTE]
// The number of references to fdEntity corresponding to fi-> fh is already incremented // The pseudo fd stored in fi->fh is attached to AutoFdEntiry so that it can be
// when it is opened. Therefore, when an existing fdEntity is detected here, the reference // destroyed here.
// count must not be incremented. And if detected, the number of references incremented
// when opened will be decremented when the AutoFdEntity object is subsequently destroyed.
// //
if(NULL == (ent = autoent.GetFdEntity(path, static_cast<int>(fi->fh), false))){ if(!autoent.Attach(path, static_cast<int>(fi->fh))){
S3FS_PRN_ERR("could not find fd(file=%s)", path); S3FS_PRN_ERR("could not find pseudo fd(file=%s, fd=%d)", path, static_cast<int>(fi->fh));
return -EIO; return -EIO;
} }
if(ent->GetFd() != static_cast<int>(fi->fh)){
S3FS_PRN_WARN("different fd(%d - %llu)", ent->GetFd(), (unsigned long long)(fi->fh));
}
} }
// check - for debug // check - for debug
if(S3fsLog::IsS3fsLogDbg()){ if(S3fsLog::IsS3fsLogDbg()){
AutoFdEntity autoent; if(FdManager::HasOpenEntityFd(path)){
FdEntity* ent; S3FS_PRN_WARN("file(%s) is still opened(another pseudo fd is opend).", path);
if(NULL != (ent = autoent.GetFdEntity(path, static_cast<int>(fi->fh)))){
S3FS_PRN_WARN("file(%s),fd(%d) is still opened.", path, ent->GetFd());
} }
} }
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -3051,7 +3038,7 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value,
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
// get xattr and make new xattr // get xattr and make new xattr
std::string strxattr; std::string strxattr;
if(ent->GetXattr(strxattr)){ if(ent->GetXattr(strxattr)){
@ -3338,7 +3325,7 @@ static int s3fs_removexattr(const char* path, const char* name)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
if(ent->MergeOrgMeta(updatemeta)){ if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading. // meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete. // then the meta is pending and accumulated to be put after the upload is complete.