octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-11-14 00:14:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Introduced pseudo fd and separated fd for each file opening

Browse Source
This commit is contained in:
Takeshi Nakatani 2021-05-23 16:28:50 +00:00 committed by Andrew Gaul
parent 53dfd48f59
commit ac578d188e
14 changed files with 1068 additions and 573 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/Makefile.am
View File

@ -47,6 +47,8 @@ s3fs_SOURCES = \
fdcache_page.cpp \ fdcache_page.cpp \
fdcache_stat.cpp \ fdcache_stat.cpp \
fdcache_auto.cpp \ fdcache_auto.cpp \
fdcache_fdinfo.cpp \
fdcache_pseudofd.cpp \
addhead.cpp \ addhead.cpp \
sighandlers.cpp \ sighandlers.cpp \
autolock.cpp \ autolock.cpp \

61
src/curl.cpp
View File

@ -3980,67 +3980,6 @@ int S3fsCurl::MultipartHeadRequest(const char* tpath, off_t size, headers_t& met
return 0; return 0;
} }
int S3fsCurl::MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool is_copy)
{
int result;
std::string upload_id;
struct stat st;
int fd2;
etaglist_t list;
off_t remaining_bytes;
off_t chunk;
S3FS_PRN_INFO3("[tpath=%s][fd=%d]", SAFESTRPTR(tpath), fd);
// duplicate fd
if(-1 == (fd2 = dup(fd)) || 0 != lseek(fd2, 0, SEEK_SET)){
S3FS_PRN_ERR("Could not duplicate file descriptor(errno=%d)", errno);
if(-1 != fd2){
close(fd2);
}
return -errno;
}
if(-1 == fstat(fd2, &st)){
S3FS_PRN_ERR("Invalid file descriptor(errno=%d)", errno);
close(fd2);
return -errno;
}
if(0 != (result = PreMultipartPostRequest(tpath, meta, upload_id, is_copy))){
close(fd2);
return result;
}
DestroyCurlHandle();
// cycle through open fd, pulling off 10MB chunks at a time
for(remaining_bytes = st.st_size; 0 < remaining_bytes; remaining_bytes -= chunk){
// chunk size
chunk = remaining_bytes > S3fsCurl::multipart_size ? S3fsCurl::multipart_size : remaining_bytes;
// set
partdata.fd = fd2;
partdata.startpos = st.st_size - remaining_bytes;
partdata.size = chunk;
b_partdata_startpos = partdata.startpos;
b_partdata_size = partdata.size;
partdata.add_etag_list(&list);
// upload part
if(0 != (result = UploadMultipartPostRequest(tpath, list.size(), upload_id))){
S3FS_PRN_ERR("failed uploading part(%d)", result);
close(fd2);
return result;
}
DestroyCurlHandle();
}
close(fd2);
if(0 != (result = CompleteMultipartPostRequest(tpath, upload_id, list))){
return result;
}
return 0;
}
int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etaglist_t& list) int S3fsCurl::MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etaglist_t& list)
{ {
S3FS_PRN_INFO3("[upload_id=%s][tpath=%s][fd=%d][offset=%lld][size=%lld]", upload_id.c_str(), SAFESTRPTR(tpath), fd, static_cast<long long int>(offset), static_cast<long long int>(size)); S3FS_PRN_INFO3("[upload_id=%s][tpath=%s][fd=%d][offset=%lld][size=%lld]", upload_id.c_str(), SAFESTRPTR(tpath), fd, static_cast<long long int>(offset), static_cast<long long int>(size));

1
src/curl.h
View File

@ -393,7 +393,6 @@ class S3fsCurl
int MultipartListRequest(std::string& body); int MultipartListRequest(std::string& body);
int AbortMultipartUpload(const char* tpath, const std::string& upload_id); int AbortMultipartUpload(const char* tpath, const std::string& upload_id);
int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy); int MultipartHeadRequest(const char* tpath, off_t size, headers_t& meta, bool is_copy);
int MultipartUploadRequest(const char* tpath, headers_t& meta, int fd, bool is_copy);
int MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etaglist_t& list); int MultipartUploadRequest(const std::string& upload_id, const char* tpath, int fd, off_t offset, off_t size, etaglist_t& list);
int MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size); int MultipartRenameRequest(const char* from, const char* to, headers_t& meta, off_t size);

159
src/fdcache.cpp
View File

@ -28,6 +28,7 @@
#include "common.h" #include "common.h"
#include "s3fs.h" #include "s3fs.h"
#include "fdcache.h" #include "fdcache.h"
#include "fdcache_pseudofd.h"
#include "s3fs_util.h" #include "s3fs_util.h"
#include "s3fs_logger.h" #include "s3fs_logger.h"
#include "string_util.h" #include "string_util.h"
@ -344,6 +345,16 @@ bool FdManager::HaveLseekHole()
return FdManager::have_lseek_hole; return FdManager::have_lseek_hole;
} }
bool FdManager::HasOpenEntityFd(const char* path)
{
FdEntity* ent;
int fd = -1;
if(NULL == (ent = FdManager::singleton.GetFdEntity(path, fd, false))){
return false;
}
return (0 < ent->GetOpenCount());
}
//------------------------------------------------ //------------------------------------------------
// FdManager methods // FdManager methods
//------------------------------------------------ //------------------------------------------------
@ -379,7 +390,7 @@ FdManager::~FdManager()
if(this == FdManager::get()){ if(this == FdManager::get()){
for(fdent_map_t::iterator iter = fent.begin(); fent.end() != iter; ++iter){ for(fdent_map_t::iterator iter = fent.begin(); fent.end() != iter; ++iter){
FdEntity* ent = (*iter).second; FdEntity* ent = (*iter).second;
S3FS_PRN_WARN("To exit with the cache file opened: path=%s, refcnt=%d", ent->GetPath(), ent->GetRefCnt()); S3FS_PRN_WARN("To exit with the cache file opened: path=%s, refcnt=%d", ent->GetPath(), ent->GetOpenCount());
delete ent; delete ent;
} }
fent.clear(); fent.clear();
@ -405,7 +416,7 @@ FdManager::~FdManager()
} }
} }
FdEntity* FdManager::GetFdEntity(const char* path, int existfd, bool increase_ref) FdEntity* FdManager::GetFdEntity(const char* path, int& existfd, bool newfd)
{ {
S3FS_PRN_INFO3("[path=%s][fd=%d]", SAFESTRPTR(path), existfd); S3FS_PRN_INFO3("[path=%s][fd=%d]", SAFESTRPTR(path), existfd);
@ -415,22 +426,29 @@ FdEntity* FdManager::GetFdEntity(const char* path, int existfd, bool increase_re
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
fdent_map_t::iterator iter = fent.find(std::string(path)); fdent_map_t::iterator iter = fent.find(std::string(path));
if(fent.end() != iter && (-1 == existfd || (*iter).second->GetFd() == existfd)){ if(fent.end() != iter && iter->second){
if(increase_ref){ if(-1 == existfd){
iter->second->Dup(); if(newfd){
existfd = iter->second->OpenPseudoFd(O_RDWR); // [NOTE] O_RDWR flags
}
return iter->second;
}else if(iter->second->FindPseudoFd(existfd)){
if(newfd){
existfd = iter->second->Dup(existfd);
}
return iter->second;
} }
return (*iter).second;
} }
if(-1 != existfd){ if(-1 != existfd){
for(iter = fent.begin(); iter != fent.end(); ++iter){ for(iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second && (*iter).second->GetFd() == existfd){ if(iter->second && iter->second->FindPseudoFd(existfd)){
// found opened fd in map // found opened fd in map
if(0 == strcmp((*iter).second->GetPath(), path)){ if(0 == strcmp(iter->second->GetPath(), path)){
if(increase_ref){ if(newfd){
iter->second->Dup(); existfd = iter->second->Dup(existfd);
} }
return (*iter).second; return iter->second;
} }
// found fd, but it is used another file(file descriptor is recycled) // found fd, but it is used another file(file descriptor is recycled)
// so returns NULL. // so returns NULL.
@ -438,24 +456,30 @@ FdEntity* FdManager::GetFdEntity(const char* path, int existfd, bool increase_re
} }
} }
} }
// If the cache directory is not specified, s3fs opens a temporary file
// when the file is opened.
if(!FdManager::IsCacheDir()){
for(iter = fent.begin(); iter != fent.end(); ++iter){
if(iter->second && iter->second->IsOpen() && 0 == strcmp(iter->second->GetPath(), path)){
return iter->second;
}
}
}
return NULL; return NULL;
} }
FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t time, bool force_tmpfile, bool is_create, bool no_fd_lock_wait) FdEntity* FdManager::Open(int& fd, const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, bool no_fd_lock_wait)
{ {
S3FS_PRN_DBG("[path=%s][size=%lld][time=%lld]", SAFESTRPTR(path), static_cast<long long>(size), static_cast<long long>(time)); S3FS_PRN_DBG("[path=%s][size=%lld][time=%lld][flags=0x%x]", SAFESTRPTR(path), static_cast<long long>(size), static_cast<long long>(time), flags);
if(!path || '\0' == path[0]){ if(!path || '\0' == path[0]){
return NULL; return NULL;
} }
bool close = false;
FdEntity* ent;
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
// search in mapping by key(path) // search in mapping by key(path)
fdent_map_t::iterator iter = fent.find(std::string(path)); fdent_map_t::iterator iter = fent.find(std::string(path));
if(fent.end() == iter && !force_tmpfile && !FdManager::IsCacheDir()){ if(fent.end() == iter && !force_tmpfile && !FdManager::IsCacheDir()){
// If the cache directory is not specified, s3fs opens a temporary file // If the cache directory is not specified, s3fs opens a temporary file
// when the file is opened. // when the file is opened.
@ -463,16 +487,17 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t
// search a entity in all which opened the temporary file. // search a entity in all which opened the temporary file.
// //
for(iter = fent.begin(); iter != fent.end(); ++iter){ for(iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second && (*iter).second->IsOpen() && 0 == strcmp((*iter).second->GetPath(), path)){ if(iter->second && iter->second->IsOpen() && 0 == strcmp(iter->second->GetPath(), path)){
break; // found opened fd in mapping break; // found opened fd in mapping
} }
} }
} }
FdEntity* ent;
if(fent.end() != iter){ if(fent.end() != iter){
// found // found
ent = (*iter).second; ent = iter->second;
ent->Dup();
if(ent->IsModified()){ if(ent->IsModified()){
// If the file is being modified and it's size is larger than size parameter, it will not be resized. // If the file is being modified and it's size is larger than size parameter, it will not be resized.
off_t cur_size = 0; off_t cur_size = 0;
@ -480,7 +505,12 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t
size = -1; size = -1;
} }
} }
close = true;
// (re)open
if(-1 == (fd = ent->Open(pmeta, size, time, flags, no_fd_lock_wait ? AutoLock::NO_WAIT : AutoLock::NON))){
S3FS_PRN_ERR("failed to (re)open and create new pseudo fd for path(%s).", path);
return NULL;
}
}else if(is_create){ }else if(is_create){
// not found // not found
@ -492,6 +522,12 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t
// make new obj // make new obj
ent = new FdEntity(path, cache_path.c_str()); ent = new FdEntity(path, cache_path.c_str());
// open
if(-1 == (fd = ent->Open(pmeta, size, time, flags, no_fd_lock_wait ? AutoLock::NO_WAIT : AutoLock::NON))){
delete ent;
return NULL;
}
if(!cache_path.empty()){ if(!cache_path.empty()){
// using cache // using cache
fent[std::string(path)] = ent; fent[std::string(path)] = ent;
@ -507,48 +543,44 @@ FdEntity* FdManager::Open(const char* path, headers_t* pmeta, off_t size, time_t
FdManager::MakeRandomTempPath(path, tmppath); FdManager::MakeRandomTempPath(path, tmppath);
fent[tmppath] = ent; fent[tmppath] = ent;
} }
}else{ }else{
return NULL; return NULL;
} }
// open
if(0 != ent->Open(pmeta, size, time, no_fd_lock_wait ? AutoLock::NO_WAIT : AutoLock::NONE)){
if(close){
ent->Close();
}
return NULL;
}
if(close){
ent->Close();
}
return ent; return ent;
} }
FdEntity* FdManager::ExistOpen(const char* path, int existfd, bool ignore_existfd) // [NOTE]
// This method does not create a new pseudo fd.
// It just finds existfd and returns the corresponding entity.
//
FdEntity* FdManager::GetExistFdEntiy(const char* path, int existfd)
{ {
S3FS_PRN_DBG("[path=%s][fd=%d][ignore_existfd=%s]", SAFESTRPTR(path), existfd, ignore_existfd ? "true" : "false"); S3FS_PRN_DBG("[path=%s][existfd=%d]", SAFESTRPTR(path), existfd);
// search by real path AutoLock auto_lock(&FdManager::fd_manager_lock);
FdEntity* ent = Open(path, NULL, -1, -1, false, false);
if(!ent && (ignore_existfd || (-1 != existfd))){ // search from all entity.
// search from all fdentity because of not using cache. for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){
AutoLock auto_lock(&FdManager::fd_manager_lock); if(iter->second && iter->second->FindPseudoFd(existfd)){
// found existfd in entity
for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){ return iter->second;
if((*iter).second && (*iter).second->IsOpen() && (ignore_existfd || ((*iter).second->GetFd() == existfd))){
// found opened fd in map
if(0 == strcmp((*iter).second->GetPath(), path)){
ent = (*iter).second;
ent->Dup();
}else{
// found fd, but it is used another file(file descriptor is recycled)
// so returns NULL.
}
break;
}
} }
} }
// not found entity
return NULL;
}
FdEntity* FdManager::OpenExistFdEntiy(const char* path, int& fd, int flags)
{
S3FS_PRN_DBG("[path=%s][flags=0x%x]", SAFESTRPTR(path), flags);
// search entity by path, and create pseudo fd
FdEntity* ent = Open(fd, path, NULL, -1, -1, flags, false, false);
if(!ent){
// Not found entity
return NULL;
}
return ent; return ent;
} }
@ -564,7 +596,7 @@ void FdManager::Rename(const std::string &from, const std::string &to)
// search a entity in all which opened the temporary file. // search a entity in all which opened the temporary file.
// //
for(iter = fent.begin(); iter != fent.end(); ++iter){ for(iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second && (*iter).second->IsOpen() && 0 == strcmp((*iter).second->GetPath(), from.c_str())){ if(iter->second && iter->second->IsOpen() && 0 == strcmp(iter->second->GetPath(), from.c_str())){
break; // found opened fd in mapping break; // found opened fd in mapping
} }
} }
@ -574,7 +606,7 @@ void FdManager::Rename(const std::string &from, const std::string &to)
// found // found
S3FS_PRN_DBG("[from=%s][to=%s]", from.c_str(), to.c_str()); S3FS_PRN_DBG("[from=%s][to=%s]", from.c_str(), to.c_str());
FdEntity* ent = (*iter).second; FdEntity* ent = iter->second;
// retrieve old fd entity from map // retrieve old fd entity from map
fent.erase(iter); fent.erase(iter);
@ -591,34 +623,33 @@ void FdManager::Rename(const std::string &from, const std::string &to)
} }
} }
bool FdManager::Close(FdEntity* ent) bool FdManager::Close(FdEntity* ent, int fd)
{ {
S3FS_PRN_DBG("[ent->file=%s][ent->fd=%d]", ent ? ent->GetPath() : "", ent ? ent->GetFd() : -1); S3FS_PRN_DBG("[ent->file=%s][pseudo_fd=%d]", ent ? ent->GetPath() : "", fd);
if(!ent){ if(!ent || -1 == fd){
return true; // returns success return true; // returns success
} }
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){ for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ++iter){
if((*iter).second == ent){ if(iter->second == ent){
ent->Close(); ent->Close(fd);
if(!ent->IsOpen()){ if(!ent->IsOpen()){
// remove found entity from map. // remove found entity from map.
fent.erase(iter++); fent.erase(iter++);
// check another key name for entity value to be on the safe side // check another key name for entity value to be on the safe side
for(; iter != fent.end(); ){ for(; iter != fent.end(); ){
if((*iter).second == ent){ if(iter->second == ent){
fent.erase(iter++); fent.erase(iter++);
}else{ }else{
++iter; ++iter;
} }
} }
delete ent; delete ent;
} }
return true; return true;
} }
} }
return false; return false;
@ -629,7 +660,7 @@ bool FdManager::ChangeEntityToTempPath(FdEntity* ent, const char* path)
AutoLock auto_lock(&FdManager::fd_manager_lock); AutoLock auto_lock(&FdManager::fd_manager_lock);
for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ){ for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); ){
if((*iter).second == ent){ if(iter->second == ent){
fent.erase(iter++); fent.erase(iter++);
std::string tmppath; std::string tmppath;

11
src/fdcache.h
View File

@ -67,7 +67,7 @@ class FdManager
static bool MakeRandomTempPath(const char* path, std::string& tmppath); static bool MakeRandomTempPath(const char* path, std::string& tmppath);
static bool SetCheckCacheDirExist(bool is_check); static bool SetCheckCacheDirExist(bool is_check);
static bool CheckCacheDirExist(); static bool CheckCacheDirExist();
static bool HasOpenEntityFd(const char* path);
static off_t GetEnsureFreeDiskSpace(); static off_t GetEnsureFreeDiskSpace();
static off_t SetEnsureFreeDiskSpace(off_t size); static off_t SetEnsureFreeDiskSpace(off_t size);
static bool IsSafeDiskSpace(const char* path, off_t size); static bool IsSafeDiskSpace(const char* path, off_t size);
@ -76,11 +76,12 @@ class FdManager
static bool HaveLseekHole(); static bool HaveLseekHole();
// Return FdEntity associated with path, returning NULL on error. This operation increments the reference count; callers must decrement via Close after use. // Return FdEntity associated with path, returning NULL on error. This operation increments the reference count; callers must decrement via Close after use.
FdEntity* GetFdEntity(const char* path, int existfd = -1, bool increase_ref = true); FdEntity* GetFdEntity(const char* path, int& existfd, bool newfd = true);
FdEntity* Open(const char* path, headers_t* pmeta = NULL, off_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false); FdEntity* Open(int& fd, const char* path, headers_t* pmeta = NULL, off_t size = -1, time_t time = -1, int flags = O_RDONLY, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false);
FdEntity* ExistOpen(const char* path, int existfd = -1, bool ignore_existfd = false); FdEntity* GetExistFdEntiy(const char* path, int existfd = -1);
FdEntity* OpenExistFdEntiy(const char* path, int& fd, int flags = O_RDONLY);
void Rename(const std::string &from, const std::string &to); void Rename(const std::string &from, const std::string &to);
bool Close(FdEntity* ent); bool Close(FdEntity* ent, int fd);
bool ChangeEntityToTempPath(FdEntity* ent, const char* path); bool ChangeEntityToTempPath(FdEntity* ent, const char* path);
void CleanupCacheDir(); void CleanupCacheDir();

79
src/fdcache_auto.cpp
View File

@ -29,7 +29,7 @@
//------------------------------------------------ //------------------------------------------------
// AutoFdEntity methods // AutoFdEntity methods
//------------------------------------------------ //------------------------------------------------
AutoFdEntity::AutoFdEntity() : pFdEntity(NULL) AutoFdEntity::AutoFdEntity() : pFdEntity(NULL), pseudo_fd(-1)
{ {
} }
@ -38,13 +38,16 @@ AutoFdEntity::AutoFdEntity() : pFdEntity(NULL)
// Even if it is called, the consistency of the number of // Even if it is called, the consistency of the number of
// references can be maintained, but this case is not assumed. // references can be maintained, but this case is not assumed.
// //
AutoFdEntity::AutoFdEntity(AutoFdEntity& other) : pFdEntity(NULL) AutoFdEntity::AutoFdEntity(AutoFdEntity& other) : pFdEntity(NULL), pseudo_fd(-1)
{ {
S3FS_PRN_WARN("This method should not be called. Please check the caller."); S3FS_PRN_WARN("This method should not be called. Please check the caller.");
if(other.pFdEntity){ if(other.pFdEntity){
other.pFdEntity->Dup(); if(-1 != (pseudo_fd = other.pFdEntity->Dup(other.pseudo_fd))){
pFdEntity = other.pFdEntity; pFdEntity = other.pFdEntity;
}else{
S3FS_PRN_ERR("Failed duplicating fd in AutoFdEntity.");
}
} }
} }
@ -56,11 +59,12 @@ AutoFdEntity::~AutoFdEntity()
bool AutoFdEntity::Close() bool AutoFdEntity::Close()
{ {
if(pFdEntity){ if(pFdEntity){
if(!FdManager::get()->Close(pFdEntity)){ if(!FdManager::get()->Close(pFdEntity, pseudo_fd)){
S3FS_PRN_ERR("Failed to close fdentity."); S3FS_PRN_ERR("Failed to close fdentity.");
return false; return false;
} }
pFdEntity = NULL; pFdEntity = NULL;
pseudo_fd = -1;
} }
return true; return true;
} }
@ -69,48 +73,61 @@ bool AutoFdEntity::Close()
// This method touches the internal fdentity with. // This method touches the internal fdentity with.
// This is used to keep the file open. // This is used to keep the file open.
// //
bool AutoFdEntity::Detach() int AutoFdEntity::Detach()
{ {
if(!pFdEntity){ if(!pFdEntity){
S3FS_PRN_ERR("Does not have a associated FdEntity."); S3FS_PRN_ERR("Does not have a associated FdEntity.");
return -1;
}
int fd = pseudo_fd;
pseudo_fd = -1;
pFdEntity = NULL;
return fd;
}
bool AutoFdEntity::Attach(const char* path, int existfd)
{
Close();
if(NULL == (pFdEntity = FdManager::get()->GetFdEntity(path, existfd, false))){
S3FS_PRN_DBG("Could not find fd entity object(file=%s, existfd=%d)", path, existfd);
return false; return false;
} }
pFdEntity = NULL; pseudo_fd = existfd;
return true; return true;
} }
FdEntity* AutoFdEntity::Open(const char* path, headers_t* pmeta, off_t size, time_t time, int flags, bool force_tmpfile, bool is_create, bool no_fd_lock_wait)
{
Close();
if(NULL == (pFdEntity = FdManager::get()->Open(pseudo_fd, path, pmeta, size, time, flags, force_tmpfile, is_create, no_fd_lock_wait))){
pseudo_fd = -1;
return NULL;
}
return pFdEntity;
}
// [NOTE] // [NOTE]
// This method calls the FdManager method without incrementing the // the fd obtained by this method is not a newly created pseudo fd.
// reference count.
// This means that it will only be used to map to a file descriptor
// that was already open.
// //
FdEntity* AutoFdEntity::GetFdEntity(const char* path, int existfd, bool increase_ref) FdEntity* AutoFdEntity::GetExistFdEntiy(const char* path, int existfd)
{ {
Close(); Close();
if(NULL == (pFdEntity = FdManager::get()->GetFdEntity(path, existfd, increase_ref))){ FdEntity* ent;
S3FS_PRN_DBG("Could not find fd(file=%s, existfd=%d)", path, existfd); if(NULL == (ent = FdManager::get()->GetExistFdEntiy(path, existfd))){
return NULL; return NULL;
} }
return pFdEntity; return ent;
} }
FdEntity* AutoFdEntity::Open(const char* path, headers_t* pmeta, off_t size, time_t time, bool force_tmpfile, bool is_create, bool no_fd_lock_wait) FdEntity* AutoFdEntity::OpenExistFdEntiy(const char* path, int flags)
{ {
Close(); Close();
if(NULL == (pFdEntity = FdManager::get()->Open(path, pmeta, size, time, force_tmpfile, is_create, no_fd_lock_wait))){ if(NULL == (pFdEntity = FdManager::get()->OpenExistFdEntiy(path, pseudo_fd, flags))){
return NULL;
}
return pFdEntity;
}
FdEntity* AutoFdEntity::ExistOpen(const char* path, int existfd, bool ignore_existfd)
{
Close();
if(NULL == (pFdEntity = FdManager::get()->ExistOpen(path, existfd, ignore_existfd))){
return NULL; return NULL;
} }
return pFdEntity; return pFdEntity;
@ -128,8 +145,12 @@ bool AutoFdEntity::operator=(AutoFdEntity& other)
Close(); Close();
if(other.pFdEntity){ if(other.pFdEntity){
other.pFdEntity->Dup(); if(-1 != (pseudo_fd = other.pFdEntity->Dup(other.pseudo_fd))){
pFdEntity = other.pFdEntity; pFdEntity = other.pFdEntity;
}else{
S3FS_PRN_ERR("Failed duplicating fd in AutoFdEntity.");
return false;
}
} }
return true; return true;
} }

12
src/fdcache_auto.h
View File

@ -35,6 +35,7 @@ class AutoFdEntity
{ {
private: private:
FdEntity* pFdEntity; FdEntity* pFdEntity;
int pseudo_fd;
private: private:
AutoFdEntity(AutoFdEntity& other); AutoFdEntity(AutoFdEntity& other);
@ -45,10 +46,13 @@ class AutoFdEntity
~AutoFdEntity(); ~AutoFdEntity();
bool Close(); bool Close();
bool Detach(); int Detach();
FdEntity* GetFdEntity(const char* path, int existfd = -1, bool increase_ref = true); bool Attach(const char* path, int existfd);
FdEntity* Open(const char* path, headers_t* pmeta = NULL, off_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false); int GetPseudoFd() const { return pseudo_fd; }
FdEntity* ExistOpen(const char* path, int existfd = -1, bool ignore_existfd = false);
FdEntity* Open(const char* path, headers_t* pmeta = NULL, off_t size = -1, time_t time = -1, int flags = O_RDONLY, bool force_tmpfile = false, bool is_create = true, bool no_fd_lock_wait = false);
FdEntity* GetExistFdEntiy(const char* path, int existfd = -1);
FdEntity* OpenExistFdEntiy(const char* path, int flags = O_RDONLY);
}; };
#endif // S3FS_FDCACHE_AUTO_H_ #endif // S3FS_FDCACHE_AUTO_H_

792
src/fdcache_entity.cpp
View File

File diff suppressed because it is too large Load Diff

33
src/fdcache_entity.h
View File

@ -23,6 +23,7 @@
#include "autolock.h" #include "autolock.h"
#include "fdcache_page.h" #include "fdcache_page.h"
#include "fdcache_fdinfo.h"
#include "metaheader.h" #include "metaheader.h"
//------------------------------------------------ //------------------------------------------------
@ -35,9 +36,9 @@ class FdEntity
pthread_mutex_t fdent_lock; pthread_mutex_t fdent_lock;
bool is_lock_init; bool is_lock_init;
int refcnt; // reference count
std::string path; // object path std::string path; // object path
int fd; // file descriptor(tmp file or cache file) int physical_fd; // physical file(cache or temporary file) descriptor
fdinfo_map_t pseudo_fd_map; // pseudo file descriptor information map
FILE* pfile; // file pointer(tmp file or cache file) FILE* pfile; // file pointer(tmp file or cache file)
ino_t inode; // inode number for cache file ino_t inode; // inode number for cache file
headers_t orgmeta; // original headers at opening headers_t orgmeta; // original headers at opening
@ -62,6 +63,8 @@ class FdEntity
void Clear(); void Clear();
ino_t GetInode(); ino_t GetInode();
int OpenMirrorFile(); int OpenMirrorFile();
int NoCacheLoadAndPost(off_t start = 0, off_t size = 0); // size=0 means loading to end
bool CheckPseudoFdFlags(int fd, bool writable, bool lock_already_held = false);
bool SetAllStatus(bool is_loaded); // [NOTE] not locking bool SetAllStatus(bool is_loaded); // [NOTE] not locking
bool SetAllStatusUnloaded() { return SetAllStatus(false); } bool SetAllStatusUnloaded() { return SetAllStatus(false); }
int UploadPendingMeta(); int UploadPendingMeta();
@ -73,16 +76,17 @@ class FdEntity
explicit FdEntity(const char* tpath = NULL, const char* cpath = NULL); explicit FdEntity(const char* tpath = NULL, const char* cpath = NULL);
~FdEntity(); ~FdEntity();
void Close(); void Close(int fd);
bool IsOpen() const { return (-1 != fd); } bool IsOpen() const { return (-1 != physical_fd); }
int Open(headers_t* pmeta, off_t size, time_t time, AutoLock::Type type); bool FindPseudoFd(int fd, bool lock_already_held = false);
bool OpenAndLoadAll(headers_t* pmeta = NULL, off_t* size = NULL, bool force_load = false); int Open(headers_t* pmeta, off_t size, time_t time, int flags, AutoLock::Type type);
int Dup(bool lock_already_held = false); bool LoadAll(int fd, headers_t* pmeta = NULL, off_t* size = NULL, bool force_load = false);
int GetRefCnt() const { return refcnt; } // [NOTE] Use only debugging int Dup(int fd, bool lock_already_held = false);
int OpenPseudoFd(int flags = O_RDONLY, bool lock_already_held = false);
int GetOpenCount(bool lock_already_held = false);
const char* GetPath() const { return path.c_str(); } const char* GetPath() const { return path.c_str(); }
bool RenamePath(const std::string& newpath, std::string& fentmapkey); bool RenamePath(const std::string& newpath, std::string& fentmapkey);
int GetFd() const { return fd; } int GetPhysicalFd() const { return physical_fd; }
bool IsModified() const; bool IsModified() const;
bool MergeOrgMeta(headers_t& updatemeta); bool MergeOrgMeta(headers_t& updatemeta);
@ -105,17 +109,16 @@ class FdEntity
bool SetContentType(const char* path); bool SetContentType(const char* path);
int Load(off_t start = 0, off_t size = 0, bool lock_already_held = false, bool is_modified_flag = false); // size=0 means loading to end int Load(off_t start = 0, off_t size = 0, bool lock_already_held = false, bool is_modified_flag = false); // size=0 means loading to end
int NoCacheLoadAndPost(off_t start = 0, off_t size = 0); // size=0 means loading to end
int NoCachePreMultipartPost(); int NoCachePreMultipartPost();
int NoCacheMultipartPost(int tgfd, off_t start, off_t size); int NoCacheMultipartPost(int tgfd, off_t start, off_t size);
int NoCacheCompleteMultipartPost(); int NoCacheCompleteMultipartPost();
off_t BytesModified(); off_t BytesModified();
int RowFlush(const char* tpath, bool force_sync = false); int RowFlush(int fd, const char* tpath, bool force_sync = false);
int Flush(bool force_sync = false) { return RowFlush(NULL, force_sync); } int Flush(int fd, bool force_sync = false) { return RowFlush(fd, NULL, force_sync); }
ssize_t Read(char* bytes, off_t start, size_t size, bool force_load = false); ssize_t Read(int fd, char* bytes, off_t start, size_t size, bool force_load = false);
ssize_t Write(const char* bytes, off_t start, size_t size); ssize_t Write(int fd, const char* bytes, off_t start, size_t size);
bool ReserveDiskSpace(off_t size); bool ReserveDiskSpace(off_t size);
bool PunchHole(off_t start = 0, size_t size = 0); bool PunchHole(off_t start = 0, size_t size = 0);

98
src/fdcache_fdinfo.cpp Normal file
View File

@ -0,0 +1,98 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Takeshi Nakatani <ggtakec.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <cstdio>
#include <cstdlib>
#include <algorithm>
#include "common.h"
#include "s3fs.h"
#include "fdcache_fdinfo.h"
#include "fdcache_pseudofd.h"
#include "autolock.h"
//------------------------------------------------
// PseudoFdInfo methods
//------------------------------------------------
PseudoFdInfo::PseudoFdInfo(int fd, int open_flags) : pseudo_fd(-1), physical_fd(fd), flags(0) //, is_lock_init(false)
{
if(-1 != physical_fd){
pseudo_fd = PseudoFdManager::Get();
flags = open_flags;
}
}
PseudoFdInfo::~PseudoFdInfo()
{
Clear();
}
bool PseudoFdInfo::Clear()
{
if(-1 != pseudo_fd){
PseudoFdManager::Release(pseudo_fd);
}
pseudo_fd = -1;
physical_fd = -1;
return true;
}
bool PseudoFdInfo::Set(int fd, int open_flags)
{
if(-1 == fd){
return false;
}
Clear();
physical_fd = fd;
pseudo_fd = PseudoFdManager::Get();
flags = open_flags;
return true;
}
bool PseudoFdInfo::Writable() const
{
if(-1 == pseudo_fd){
return false;
}
if(0 == (flags & (O_WRONLY | O_RDWR))){
return false;
}
return true;
}
bool PseudoFdInfo::Readable() const
{
if(-1 == pseudo_fd){
return false;
}
// O_RDONLY is 0x00, it means any pattern is readable.
return true;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

61
src/fdcache_fdinfo.h Normal file
View File

@ -0,0 +1,61 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_FDCACHE_FDINFO_H_
#define S3FS_FDCACHE_FDINFO_H_
//------------------------------------------------
// Class PseudoFdInfo
//------------------------------------------------
class PseudoFdInfo
{
private:
int pseudo_fd;
int physical_fd;
int flags; // flags at open
private:
bool Clear();
public:
PseudoFdInfo(int fd = -1, int open_flags = 0);
~PseudoFdInfo();
int GetPhysicalFd() const { return physical_fd; }
int GetPseudoFd() const { return pseudo_fd; }
int GetFlags() const { return flags; }
bool Writable() const;
bool Readable() const;
bool Set(int fd, int open_flags);
};
typedef std::map<int, class PseudoFdInfo*> fdinfo_map_t;
#endif // S3FS_FDCACHE_FDINFO_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

134
src/fdcache_pseudofd.cpp Normal file
View File

@ -0,0 +1,134 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Takeshi Nakatani <ggtakec.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#include <cstdio>
#include <cstdlib>
#include <algorithm>
#include "common.h"
#include "s3fs.h"
#include "fdcache_pseudofd.h"
#include "autolock.h"
//------------------------------------------------
// Symbols
//------------------------------------------------
// [NOTE]
// The minimum pseudo fd value starts 2.
// This is to avoid mistakes for 0(stdout) and 1(stderr), which are usually used.
//
#define MIN_PSEUDOFD_NUMBER 2
//------------------------------------------------
// PseudoFdManager class methods
//------------------------------------------------
PseudoFdManager& PseudoFdManager::GetManager()
{
static PseudoFdManager singleton;
return singleton;
}
int PseudoFdManager::Get()
{
return (PseudoFdManager::GetManager()).CreatePseudoFd();
}
bool PseudoFdManager::Release(int fd)
{
return (PseudoFdManager::GetManager()).ReleasePseudoFd(fd);
}
//------------------------------------------------
// PseudoFdManager methods
//------------------------------------------------
PseudoFdManager::PseudoFdManager() : is_lock_init(false)
{
pthread_mutexattr_t attr;
pthread_mutexattr_init(&attr);
#if S3FS_PTHREAD_ERRORCHECK
pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
#endif
int result;
if(0 != (result = pthread_mutex_init(&pseudofd_list_lock, &attr))){
S3FS_PRN_CRIT("failed to init pseudofd_list_lock: %d", result);
abort();
}
is_lock_init = true;
}
PseudoFdManager::~PseudoFdManager()
{
if(is_lock_init){
int result;
if(0 != (result = pthread_mutex_destroy(&pseudofd_list_lock))){
S3FS_PRN_CRIT("failed to destroy pseudofd_list_lock: %d", result);
abort();
}
is_lock_init = false;
}
}
int PseudoFdManager::GetUnusedMinPseudoFd() const
{
int min_fd = MIN_PSEUDOFD_NUMBER;
// Look for the first discontinuous value.
for(pseudofd_list_t::const_iterator iter = pseudofd_list.begin(); iter != pseudofd_list.end(); ++iter){
if(min_fd == (*iter)){
++min_fd;
}else if(min_fd < (*iter)){
break;
}
}
return min_fd;
}
int PseudoFdManager::CreatePseudoFd()
{
AutoLock auto_lock(&pseudofd_list_lock);
int new_fd = PseudoFdManager::GetUnusedMinPseudoFd();
pseudofd_list.push_back(new_fd);
std::sort(pseudofd_list.begin(), pseudofd_list.end());
return new_fd;
}
bool PseudoFdManager::ReleasePseudoFd(int fd)
{
AutoLock auto_lock(&pseudofd_list_lock);
for(pseudofd_list_t::iterator iter = pseudofd_list.begin(); iter != pseudofd_list.end(); ++iter){
if(fd == (*iter)){
pseudofd_list.erase(iter);
return true;
}
}
return false;
}
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

65
src/fdcache_pseudofd.h Normal file
View File

@ -0,0 +1,65 @@
/*
* s3fs - FUSE-based file system backed by Amazon S3
*
* Copyright(C) 2007 Randy Rizun <rrizun@gmail.com>
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
#ifndef S3FS_FDCACHE_PSEUDOFD_H_
#define S3FS_FDCACHE_PSEUDOFD_H_
//------------------------------------------------
// Typdefs
//------------------------------------------------
// List of pseudo fd in use
//
typedef std::vector<int> pseudofd_list_t;
//------------------------------------------------
// Class PseudoFdManager
//------------------------------------------------
class PseudoFdManager
{
private:
pseudofd_list_t pseudofd_list;
bool is_lock_init;
pthread_mutex_t pseudofd_list_lock; // protects pseudofd_list
private:
static PseudoFdManager& GetManager();
PseudoFdManager();
~PseudoFdManager();
int GetUnusedMinPseudoFd() const;
int CreatePseudoFd();
bool ReleasePseudoFd(int fd);
public:
static int Get();
static bool Release(int fd);
};
#endif // S3FS_FDCACHE_PSEUDOFD_H_
/*
* Local variables:
* tab-width: 4
* c-basic-offset: 4
* End:
* vim600: expandtab sw=4 ts=4 fdm=marker
* vim<600: expandtab sw=4 ts=4
*/

133
src/s3fs.cpp
View File

@ -119,7 +119,7 @@ static int get_object_attribute(const char* path, struct stat* pstbuf, headers_t
static int check_object_access(const char* path, int mask, struct stat* pstbuf); static int check_object_access(const char* path, int mask, struct stat* pstbuf);
static int check_object_owner(const char* path, struct stat* pstbuf); static int check_object_owner(const char* path, struct stat* pstbuf);
static int check_parent_object_access(const char* path, int mask); static int check_parent_object_access(const char* path, int mask);
static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, bool is_load = false); static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, int flags = O_RDONLY, bool is_load = false);
static bool multi_head_callback(S3fsCurl* s3fscurl); static bool multi_head_callback(S3fsCurl* s3fscurl);
static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl); static S3fsCurl* multi_head_retry_callback(S3fsCurl* s3fscurl);
static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf, fuse_fill_dir_t filler); static int readdir_multi_head(const char* path, const S3ObjList& head, void* buf, fuse_fill_dir_t filler);
@ -700,7 +700,7 @@ bool get_object_sse_type(const char* path, sse_type_t& ssetype, std::string& sse
return true; return true;
} }
static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, bool is_load) static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char* path, int flags, bool is_load)
{ {
int result; int result;
struct stat stobj; struct stat stobj;
@ -717,12 +717,12 @@ static int get_local_fent(AutoFdEntity& autoent, FdEntity **entity, const char*
time_t mtime = (!S_ISREG(stobj.st_mode) && !S_ISLNK(stobj.st_mode)) ? -1 : stobj.st_mtime; time_t mtime = (!S_ISREG(stobj.st_mode) && !S_ISLNK(stobj.st_mode)) ? -1 : stobj.st_mtime;
bool force_tmpfile = S_ISREG(stobj.st_mode) ? false : true; bool force_tmpfile = S_ISREG(stobj.st_mode) ? false : true;
if(NULL == (ent = autoent.Open(path, &meta, stobj.st_size, mtime, force_tmpfile, true))){ if(NULL == (ent = autoent.Open(path, &meta, stobj.st_size, mtime, flags, force_tmpfile, true))){
S3FS_PRN_ERR("Could not open file. errno(%d)", errno); S3FS_PRN_ERR("Could not open file. errno(%d)", errno);
return -EIO; return -EIO;
} }
// load // load
if(is_load && !ent->OpenAndLoadAll(&meta)){ if(is_load && !ent->LoadAll(autoent.GetPseudoFd(), &meta)){
S3FS_PRN_ERR("Could not load file. errno(%d)", errno); S3FS_PRN_ERR("Could not load file. errno(%d)", errno);
autoent.Close(); autoent.Close();
return -EIO; return -EIO;
@ -780,7 +780,7 @@ static int s3fs_getattr(const char* _path, struct stat* stbuf)
if(stbuf){ if(stbuf){
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL != (ent = autoent.ExistOpen(path))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
struct stat tmpstbuf; struct stat tmpstbuf;
if(ent->GetStats(tmpstbuf)){ if(ent->GetStats(tmpstbuf)){
stbuf->st_size = tmpstbuf.st_size; stbuf->st_size = tmpstbuf.st_size;
@ -811,7 +811,7 @@ static int s3fs_readlink(const char* _path, char* buf, size_t size)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
int result; int result;
if(0 != (result = get_local_fent(autoent, &ent, path))){ if(0 != (result = get_local_fent(autoent, &ent, path, O_RDONLY))){
S3FS_PRN_ERR("could not get fent(file=%s)", path); S3FS_PRN_ERR("could not get fent(file=%s)", path);
return result; return result;
} }
@ -826,7 +826,7 @@ static int s3fs_readlink(const char* _path, char* buf, size_t size)
} }
// Read // Read
ssize_t ressize; ssize_t ressize;
if(0 > (ressize = ent->Read(buf, 0, readsize))){ if(0 > (ressize = ent->Read(autoent.GetPseudoFd(), buf, 0, readsize))){
S3FS_PRN_ERR("could not read file(file=%s, ressize=%zd)", path, ressize); S3FS_PRN_ERR("could not read file(file=%s, ressize=%zd)", path, ressize);
return static_cast<int>(ressize); return static_cast<int>(ressize);
} }
@ -994,13 +994,12 @@ static int s3fs_create(const char* _path, mode_t mode, struct fuse_file_info* fi
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.Open(path, &meta, 0, -1, false, true))){ if(NULL == (ent = autoent.Open(path, &meta, 0, -1, fi->flags, false, true))){
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
return -EIO; return -EIO;
} }
ent->MarkDirtyNewFile(); ent->MarkDirtyNewFile();
autoent.Detach(); // KEEP fdentity open fi->fh = autoent.Detach(); // KEEP fdentity open;
fi->fh = ent->GetFd();
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -1191,19 +1190,25 @@ static int s3fs_symlink(const char* _from, const char* _to)
{ // scope for AutoFdEntity { // scope for AutoFdEntity
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.Open(to, &headers, 0, -1, true, true))){ if(NULL == (ent = autoent.Open(to, &headers, 0, -1, O_RDWR, true, true))){
S3FS_PRN_ERR("could not open tmpfile(errno=%d)", errno); S3FS_PRN_ERR("could not open tmpfile(errno=%d)", errno);
return -errno; return -errno;
} }
// write(without space words) // write(without space words)
strFrom = trim(std::string(from)); strFrom = trim(std::string(from));
ssize_t from_size = static_cast<ssize_t>(strFrom.length()); ssize_t from_size = static_cast<ssize_t>(strFrom.length());
if(from_size != ent->Write(strFrom.c_str(), 0, from_size)){ ssize_t ressize;
S3FS_PRN_ERR("could not write tmpfile(errno=%d)", errno); if(from_size != (ressize = ent->Write(autoent.GetPseudoFd(), strFrom.c_str(), 0, from_size))){
return -errno; if(ressize < 0){
S3FS_PRN_ERR("could not write tmpfile(errno=%d)", static_cast<int>(ressize));
return static_cast<int>(ressize);
}else{
S3FS_PRN_ERR("could not write tmpfile %zd byte(errno=%d)", ressize, errno);
return (0 == errno ? -EIO : -errno);
}
} }
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_WARN("could not upload tmpfile(result=%d)", result); S3FS_PRN_WARN("could not upload tmpfile(result=%d)", result);
} }
} }
@ -1254,11 +1259,11 @@ static int rename_object(const char* from, const char* to, bool update_ctime)
// update time // update time
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.ExistOpen(from, -1, !FdManager::IsCacheDir()))){ if(NULL == (ent = autoent.OpenExistFdEntiy(from))){
// no opened fd // no opened fd
if(FdManager::IsCacheDir()){ if(FdManager::IsCacheDir()){
// create cache file if be needed // create cache file if be needed
ent = autoent.Open(from, &meta, buf.st_size, -1, false, true); ent = autoent.Open(from, &meta, buf.st_size, -1, O_RDONLY, false, true);
} }
if(ent){ if(ent){
struct timespec mtime = get_mtime(meta); struct timespec mtime = get_mtime(meta);
@ -1317,7 +1322,7 @@ static int rename_object_nocopy(const char* from, const char* to, bool update_ct
{ // scope for AutoFdEntity { // scope for AutoFdEntity
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(0 != (result = get_local_fent(autoent, &ent, from, true))){ if(0 != (result = get_local_fent(autoent, &ent, from, O_RDWR, true))){
S3FS_PRN_ERR("could not open and read file(%s)", from); S3FS_PRN_ERR("could not open and read file(%s)", from);
return result; return result;
} }
@ -1335,7 +1340,7 @@ static int rename_object_nocopy(const char* from, const char* to, bool update_ct
} }
// upload // upload
if(0 != (result = ent->RowFlush(to, true))){ if(0 != (result = ent->RowFlush(autoent.GetPseudoFd(), to, true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result); S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result);
return result; return result;
} }
@ -1573,8 +1578,8 @@ static int s3fs_rename(const char* _from, const char* _to)
{ // scope for AutoFdEntity { // scope for AutoFdEntity
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL != (ent = autoent.ExistOpen(from))){ if(NULL != (ent = autoent.OpenExistFdEntiy(from, O_RDWR))){
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result); S3FS_PRN_ERR("could not upload file(%s): result=%d", to, result);
return result; return result;
} }
@ -1673,7 +1678,7 @@ static int s3fs_chmod(const char* _path, mode_t mode)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
if(ent->MergeOrgMeta(updatemeta)){ if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading. // meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete. // then the meta is pending and accumulated to be put after the upload is complete.
@ -1755,7 +1760,7 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
// open & load // open & load
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), true))){ if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
return result; return result;
} }
@ -1767,7 +1772,7 @@ static int s3fs_chmod_nocopy(const char* _path, mode_t mode)
ent->SetMode(mode); ent->SetMode(mode);
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
return result; return result;
} }
@ -1850,7 +1855,7 @@ static int s3fs_chown(const char* _path, uid_t uid, gid_t gid)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
if(ent->MergeOrgMeta(updatemeta)){ if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading. // meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete. // then the meta is pending and accumulated to be put after the upload is complete.
@ -1939,7 +1944,7 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
// open & load // open & load
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), true))){ if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
return result; return result;
} }
@ -1952,7 +1957,7 @@ static int s3fs_chown_nocopy(const char* _path, uid_t uid, gid_t gid)
ent->SetGId(gid); ent->SetGId(gid);
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
return result; return result;
} }
@ -2032,7 +2037,7 @@ static int s3fs_utimens(const char* _path, const struct timespec ts[2])
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
bool keep_mtime = false; bool keep_mtime = false;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
if(ent->MergeOrgMeta(updatemeta)){ if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading. // meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete. // then the meta is pending and accumulated to be put after the upload is complete.
@ -2131,7 +2136,7 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2])
// open & load // open & load
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), true))){ if(0 != (result = get_local_fent(autoent, &ent, strpath.c_str(), O_RDWR, true))){
S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str()); S3FS_PRN_ERR("could not open and read file(%s)", strpath.c_str());
return result; return result;
} }
@ -2149,7 +2154,7 @@ static int s3fs_utimens_nocopy(const char* _path, const struct timespec ts[2])
} }
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result); S3FS_PRN_ERR("could not upload file(%s): result=%d", strpath.c_str(), result);
return result; return result;
} }
@ -2184,7 +2189,7 @@ static int s3fs_truncate(const char* _path, off_t size)
// Get file information // Get file information
if(0 == (result = get_object_attribute(path, NULL, &meta))){ if(0 == (result = get_object_attribute(path, NULL, &meta))){
// Exists -> Get file(with size) // Exists -> Get file(with size)
if(NULL == (ent = autoent.Open(path, &meta, size, -1, false, true))){ if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, false, true))){
S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno); S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno);
return -EIO; return -EIO;
} }
@ -2209,14 +2214,14 @@ static int s3fs_truncate(const char* _path, off_t size)
meta["x-amz-meta-uid"] = str(pcxt->uid); meta["x-amz-meta-uid"] = str(pcxt->uid);
meta["x-amz-meta-gid"] = str(pcxt->gid); meta["x-amz-meta-gid"] = str(pcxt->gid);
if(NULL == (ent = autoent.Open(path, &meta, size, -1, true, true))){ if(NULL == (ent = autoent.Open(path, &meta, size, -1, O_RDWR, true, true))){
S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno); S3FS_PRN_ERR("could not open file(%s): errno=%d", path, errno);
return -EIO; return -EIO;
} }
} }
// upload // upload
if(0 != (result = ent->Flush(true))){ if(0 != (result = ent->Flush(autoent.GetPseudoFd(), true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result); S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result);
return result; return result;
} }
@ -2247,11 +2252,7 @@ static int s3fs_open(const char* _path, struct fuse_file_info* fi)
// and only the Stats cache exists. // and only the Stats cache exists.
// //
if(StatCache::getStatCacheData()->HasStat(path)){ if(StatCache::getStatCacheData()->HasStat(path)){
AutoFdEntity autoent_local; if(!FdManager::HasOpenEntityFd(path)){
if(NULL == autoent_local.ExistOpen(path, -1, true)){
if((result = s3fs_flush(_path, fi)) != 0){
S3FS_PRN_ERR("could not flush(%s): result=%d", path, result);
}
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
} }
} }
@ -2284,7 +2285,7 @@ static int s3fs_open(const char* _path, struct fuse_file_info* fi)
FdEntity* ent; FdEntity* ent;
headers_t meta; headers_t meta;
get_object_attribute(path, NULL, &meta, true, NULL, true); // no truncate cache get_object_attribute(path, NULL, &meta, true, NULL, true); // no truncate cache
if(NULL == (ent = autoent.Open(path, &meta, st.st_size, st.st_mtime, false, true))){ if(NULL == (ent = autoent.Open(path, &meta, st.st_size, st.st_mtime, fi->flags, false, true))){
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
return -EIO; return -EIO;
} }
@ -2293,14 +2294,13 @@ static int s3fs_open(const char* _path, struct fuse_file_info* fi)
time_t now = time(NULL); time_t now = time(NULL);
struct timespec ts = {now, 0}; struct timespec ts = {now, 0};
ent->SetMCtime(ts, ts); ent->SetMCtime(ts, ts);
if(0 != (result = ent->RowFlush(path, true))){ if(0 != (result = ent->RowFlush(autoent.GetPseudoFd(), path, true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result); S3FS_PRN_ERR("could not upload file(%s): result=%d", path, result);
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
return result; return result;
} }
} }
autoent.Detach(); // KEEP fdentity open fi->fh = autoent.Detach(); // KEEP fdentity open;
fi->fh = ent->GetFd();
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -2316,13 +2316,10 @@ static int s3fs_read(const char* _path, char* buf, size_t size, off_t offset, st
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.ExistOpen(path, static_cast<int>(fi->fh)))){ if(NULL == (ent = autoent.GetExistFdEntiy(path, static_cast<int>(fi->fh)))){
S3FS_PRN_ERR("could not find opened fd(%s)", path); S3FS_PRN_ERR("could not find opened fd(%s)", path);
return -EIO; return -EIO;
} }
if(ent->GetFd() != static_cast<int>(fi->fh)){
S3FS_PRN_WARN("different fd(%d - %llu)", ent->GetFd(), (unsigned long long)(fi->fh));
}
// check real file size // check real file size
off_t realsize = 0; off_t realsize = 0;
@ -2331,7 +2328,7 @@ static int s3fs_read(const char* _path, char* buf, size_t size, off_t offset, st
return 0; return 0;
} }
if(0 > (res = ent->Read(buf, offset, size, false))){ if(0 > (res = ent->Read(static_cast<int>(fi->fh), buf, offset, size, false))){
S3FS_PRN_WARN("failed to read file(%s). result=%zd", path, res); S3FS_PRN_WARN("failed to read file(%s). result=%zd", path, res);
} }
@ -2347,20 +2344,18 @@ static int s3fs_write(const char* _path, const char* buf, size_t size, off_t off
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL == (ent = autoent.ExistOpen(path, static_cast<int>(fi->fh)))){ if(NULL == (ent = autoent.GetExistFdEntiy(path, static_cast<int>(fi->fh)))){
S3FS_PRN_ERR("could not find opened fd(%s)", path); S3FS_PRN_ERR("could not find opened fd(%s)", path);
return -EIO; return -EIO;
} }
if(ent->GetFd() != static_cast<int>(fi->fh)){
S3FS_PRN_WARN("different fd(%d - %llu)", ent->GetFd(), (unsigned long long)(fi->fh)); if(0 > (res = ent->Write(static_cast<int>(fi->fh), buf, offset, size))){
}
if(0 > (res = ent->Write(buf, offset, size))){
S3FS_PRN_WARN("failed to write file(%s). result=%zd", path, res); S3FS_PRN_WARN("failed to write file(%s). result=%zd", path, res);
} }
if(max_dirty_data != -1 && ent->BytesModified() >= max_dirty_data){ if(max_dirty_data != -1 && ent->BytesModified() >= max_dirty_data){
int flushres; int flushres;
if(0 != (flushres = ent->RowFlush(path, true))){ if(0 != (flushres = ent->RowFlush(autoent.GetPseudoFd(), path, true))){
S3FS_PRN_ERR("could not upload file(%s): result=%d", path, flushres); S3FS_PRN_ERR("could not upload file(%s): result=%d", path, flushres);
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
return flushres; return flushres;
@ -2408,10 +2403,10 @@ static int s3fs_flush(const char* _path, struct fuse_file_info* fi)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL != (ent = autoent.ExistOpen(path, static_cast<int>(fi->fh)))){ if(NULL != (ent = autoent.GetExistFdEntiy(path, static_cast<int>(fi->fh)))){
ent->UpdateMtime(true); // clear the flag not to update mtime. ent->UpdateMtime(true); // clear the flag not to update mtime.
ent->UpdateCtime(); ent->UpdateCtime();
result = ent->Flush(false); result = ent->Flush(static_cast<int>(fi->fh), false);
StatCache::getStatCacheData()->DelStat(path); StatCache::getStatCacheData()->DelStat(path);
} }
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -2431,12 +2426,12 @@ static int s3fs_fsync(const char* _path, int datasync, struct fuse_file_info* fi
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
if(NULL != (ent = autoent.ExistOpen(path, static_cast<int>(fi->fh)))){ if(NULL != (ent = autoent.GetExistFdEntiy(path, static_cast<int>(fi->fh)))){
if(0 == datasync){ if(0 == datasync){
ent->UpdateMtime(); ent->UpdateMtime();
ent->UpdateCtime(); ent->UpdateCtime();
} }
result = ent->Flush(false); result = ent->Flush(static_cast<int>(fi->fh), false);
} }
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -2468,29 +2463,21 @@ static int s3fs_release(const char* _path, struct fuse_file_info* fi)
{ // scope for AutoFdEntity { // scope for AutoFdEntity
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent;
// [NOTE] // [NOTE]
// The number of references to fdEntity corresponding to fi-> fh is already incremented // The pseudo fd stored in fi->fh is attached to AutoFdEntiry so that it can be
// when it is opened. Therefore, when an existing fdEntity is detected here, the reference // destroyed here.
// count must not be incremented. And if detected, the number of references incremented
// when opened will be decremented when the AutoFdEntity object is subsequently destroyed.
// //
if(NULL == (ent = autoent.GetFdEntity(path, static_cast<int>(fi->fh), false))){ if(!autoent.Attach(path, static_cast<int>(fi->fh))){
S3FS_PRN_ERR("could not find fd(file=%s)", path); S3FS_PRN_ERR("could not find pseudo fd(file=%s, fd=%d)", path, static_cast<int>(fi->fh));
return -EIO; return -EIO;
} }
if(ent->GetFd() != static_cast<int>(fi->fh)){
S3FS_PRN_WARN("different fd(%d - %llu)", ent->GetFd(), (unsigned long long)(fi->fh));
}
} }
// check - for debug // check - for debug
if(S3fsLog::IsS3fsLogDbg()){ if(S3fsLog::IsS3fsLogDbg()){
AutoFdEntity autoent; if(FdManager::HasOpenEntityFd(path)){
FdEntity* ent; S3FS_PRN_WARN("file(%s) is still opened(another pseudo fd is opend).", path);
if(NULL != (ent = autoent.GetFdEntity(path, static_cast<int>(fi->fh)))){
S3FS_PRN_WARN("file(%s),fd(%d) is still opened.", path, ent->GetFd());
} }
} }
S3FS_MALLOCTRIM(0); S3FS_MALLOCTRIM(0);
@ -3051,7 +3038,7 @@ static int s3fs_setxattr(const char* path, const char* name, const char* value,
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
// get xattr and make new xattr // get xattr and make new xattr
std::string strxattr; std::string strxattr;
if(ent->GetXattr(strxattr)){ if(ent->GetXattr(strxattr)){
@ -3338,7 +3325,7 @@ static int s3fs_removexattr(const char* path, const char* name)
AutoFdEntity autoent; AutoFdEntity autoent;
FdEntity* ent; FdEntity* ent;
bool need_put_header = true; bool need_put_header = true;
if(NULL != (ent = autoent.ExistOpen(path, -1, true))){ if(NULL != (ent = autoent.OpenExistFdEntiy(path))){
if(ent->MergeOrgMeta(updatemeta)){ if(ent->MergeOrgMeta(updatemeta)){
// meta is changed, but now uploading. // meta is changed, but now uploading.
// then the meta is pending and accumulated to be put after the upload is complete. // then the meta is pending and accumulated to be put after the upload is complete.