octoleo/s3fs-fuse
1
0
Fork 0
mirror of https://github.com/s3fs-fuse/s3fs-fuse.git synced 2024-09-19 21:09:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #138 from s3fs-fuse/issue#97

Browse Source 
Fixed bugs, not turn use_cache off and ty to load to end - issue#97
This commit is contained in:
Takeshi Nakatani 2015-03-04 17:52:22 +09:00
commit e66e5d1dfc
3 changed files with 106 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

106
src/fdcache.cpp
View File

@ -19,6 +19,7 @@
*/
#include <stdio.h>
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/time.h>
@ -51,7 +52,7 @@ using namespace std;
// Symbols
//------------------------------------------------
#define MAX_MULTIPART_CNT 10000 // S3 multipart max count
#define FDPAGE_SIZE (50 * 1024 * 1024) // 50MB(parallel uploading is 5 parallel(default) * 10 MB)
#define FDPAGE_SIZE (50 * 1024 * 1024) // 50MB(parallel uploading is 5 parallel(default) * 10 MB)
//------------------------------------------------
// CacheFileStat class methods
@ -344,10 +345,14 @@ bool PageList::FindUninitPage(off_t start, off_t& resstart, size_t& ressize)
return false;
}
int PageList::GetUninitPages(fdpage_list_t& uninit_list, off_t start)
int PageList::GetUninitPages(fdpage_list_t& uninit_list, off_t start, off_t size)
{
for(fdpage_list_t::iterator iter = pages.begin(); iter != pages.end(); iter++){
if(start <= (*iter)->end()){
if((start + size) <= (*iter)->offset){
// reach to end
break;
}
// after start pos
if(!(*iter)->init){
// found uninitialized area
@ -790,7 +795,7 @@ int FdEntity::Load(off_t start, off_t size)
// check loaded area & load
fdpage_list_t uninit_list;
if(0 < pagelist.GetUninitPages(uninit_list, start)){
if(0 < pagelist.GetUninitPages(uninit_list, start, size)){
for(fdpage_list_t::iterator iter = uninit_list.begin(); iter != uninit_list.end(); iter++){
if(-1 != size && (start + size) <= (*iter)->offset){
break;
@ -989,6 +994,24 @@ ssize_t FdEntity::Write(const char* bytes, off_t start, size_t size)
return wsize;
}
//------------------------------------------------
// FdManager symbol
//------------------------------------------------
// [NOTE]
// NOCACHE_PATH_PREFIX symbol needs for not using cache mode.
// Now s3fs I/F functions in s3fs.cpp has left the processing
// to FdManager and FdEntity class. FdManager class manages
// the list of local file stat and file discriptor in conjunction
// with the FdEntity class.
// When s3fs is not using local cache, it means FdManager must
// return new temporary file discriptor at each opening it.
// Then FdManager caches fd by key which is dummy file path
// instead of real file path.
// This process may not be complete, but it is easy way can
// be realized.
//
#define NOCACHE_PATH_PREFIX_FORM " __S3FS_UNEXISTED_PATH_%lx__ / " // important space words for simply
//------------------------------------------------
// FdManager class valiable
//------------------------------------------------
@ -1082,6 +1105,16 @@ bool FdManager::MakeCachePath(const char* path, string& cache_path, bool is_crea
return true;
}
bool FdManager::MakeRandomTempPath(const char* path, string& tmppath)
{
char szBuff[64];
sprintf(szBuff, NOCACHE_PATH_PREFIX_FORM, random()); // warry for performance, but maybe don't warry.
tmppath = szBuff;
tmppath += path ? path : "";
return true;
}
//------------------------------------------------
// FdManager methods
//------------------------------------------------
@ -1122,9 +1155,9 @@ FdManager::~FdManager()
}
}
FdEntity* FdManager::GetFdEntity(const char* path)
FdEntity* FdManager::GetFdEntity(const char* path, int existfd)
{
FPRNINFO("[path=%s]", SAFESTRPTR(path));
FPRNINFO("[path=%s][fd=%d]", SAFESTRPTR(path), existfd);
if(!path || '\0' == path[0]){
return NULL;
@ -1132,10 +1165,24 @@ FdEntity* FdManager::GetFdEntity(const char* path)
AutoLock auto_lock(&FdManager::fd_manager_lock);
fdent_map_t::iterator iter = fent.find(string(path));
if(fent.end() == iter){
return NULL;
if(fent.end() != iter && (-1 == existfd || (*iter).second->GetFd() == existfd)){
return (*iter).second;
}
return (*iter).second;
if(-1 != existfd){
for(iter = fent.begin(); iter != fent.end(); iter++){
if((*iter).second && (*iter).second->GetFd() == existfd){
// found opend fd in map
if(0 == strcmp((*iter).second->GetPath(), path)){
return (*iter).second;
}
// found fd, but it is used another file(file discriptor is recycled)
// so returns NULL.
break;
}
}
}
return NULL;
}
FdEntity* FdManager::Open(const char* path, off_t size, time_t time, bool force_tmpfile, bool is_create)
@ -1164,8 +1211,22 @@ FdEntity* FdManager::Open(const char* path, off_t size, time_t time, bool force_
}
// make new obj
ent = new FdEntity(path, cache_path.c_str());
fent[string(path)] = ent;
if(0 < cache_path.size()){
// using cache
fent[string(path)] = ent;
}else{
// not using cache, so the key of fdentity is set not really existsing path.
// (but not strictly unexisting path.)
//
// [NOTE]
// The reason why this process here, please look at the definition of the
// comments of NOCACHE_PATH_PREFIX_FORM symbol.
//
string tmppath("");
FdManager::MakeRandomTempPath(path, tmppath);
fent[tmppath] = ent;
}
}else{
return NULL;
}
@ -1177,6 +1238,33 @@ FdEntity* FdManager::Open(const char* path, off_t size, time_t time, bool force_
return ent;
}
FdEntity* FdManager::ExistOpen(const char* path, int existfd)
{
FPRNINFO("[path=%s][fd=%d]", SAFESTRPTR(path), existfd);
// search by real path
FdEntity* ent = Open(path, -1, -1, false, false);
if(!ent && -1 != existfd){
// search from all fdentity because of not using cache.
AutoLock auto_lock(&FdManager::fd_manager_lock);
for(fdent_map_t::iterator iter = fent.begin(); iter != fent.end(); iter++){
if((*iter).second && (*iter).second->GetFd() == existfd && (*iter).second->IsOpen()){
// found opend fd in map
if(0 == strcmp((*iter).second->GetPath(), path)){
ent = (*iter).second;
}else{
// found fd, but it is used another file(file discriptor is recycled)
// so returns NULL.
}
break;
}
}
}
return ent;
}
void FdManager::Rename(const std::string &from, const std::string &to)
{
fdent_map_t::iterator iter = fent.find(from);

7
src/fdcache.h
View File

@ -85,7 +85,7 @@ class PageList
bool IsInit(off_t start, off_t size);
bool SetInit(off_t start, off_t size, bool is_init = true);
bool FindUninitPage(off_t start, off_t& resstart, size_t& ressize);
int GetUninitPages(fdpage_list_t& uninit_list, off_t start = 0);
int GetUninitPages(fdpage_list_t& uninit_list, off_t start = 0, off_t size = -1);
bool Serialize(CacheFileStat& file, bool is_output);
void Dump(void);
};
@ -166,10 +166,11 @@ class FdManager
static size_t SetPageSize(size_t size);
static size_t GetPageSize(void) { return FdManager::page_size; }
static bool MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir = true);
static bool MakeRandomTempPath(const char* path, std::string& tmppath);
FdEntity* GetFdEntity(const char* path);
FdEntity* GetFdEntity(const char* path, int existfd = -1);
FdEntity* Open(const char* path, off_t size = -1, time_t time = -1, bool force_tmpfile = false, bool is_create = true);
FdEntity* ExistOpen(const char* path) { return Open(path, -1, -1, false, false); }
FdEntity* ExistOpen(const char* path, int existfd = -1);
void Rename(const std::string &from, const std::string &to);
bool Close(FdEntity* ent);
};

10
src/s3fs.cpp
View File

@ -1953,7 +1953,7 @@ static int s3fs_read(const char* path, char* buf, size_t size, off_t offset, str
FPRNINFO("[path=%s][size=%zu][offset=%jd][fd=%llu]", path, size, (intmax_t)offset, (unsigned long long)(fi->fh));
FdEntity* ent;
if(NULL == (ent = FdManager::get()->ExistOpen(path))){
if(NULL == (ent = FdManager::get()->ExistOpen(path, static_cast<int>(fi->fh)))){
DPRN("could not find opened fd(%s)", path);
return -EIO;
}
@ -1984,7 +1984,7 @@ static int s3fs_write(const char* path, const char* buf, size_t size, off_t offs
FPRNINFO("[path=%s][size=%zu][offset=%jd][fd=%llu]", path, size, (intmax_t)offset, (unsigned long long)(fi->fh));
FdEntity* ent;
if(NULL == (ent = FdManager::get()->ExistOpen(path))){
if(NULL == (ent = FdManager::get()->ExistOpen(path, static_cast<int>(fi->fh)))){
DPRN("could not find opened fd(%s)", path);
return -EIO;
}
@ -2030,7 +2030,7 @@ static int s3fs_flush(const char* path, struct fuse_file_info* fi)
}
FdEntity* ent;
if(NULL != (ent = FdManager::get()->ExistOpen(path))){
if(NULL != (ent = FdManager::get()->ExistOpen(path, static_cast<int>(fi->fh)))){
headers_t meta;
if(0 != (result = get_object_attribute(path, NULL, &meta))){
FdManager::get()->Close(ent);
@ -2067,7 +2067,7 @@ static int s3fs_release(const char* path, struct fuse_file_info* fi)
}
FdEntity* ent;
if(NULL == (ent = FdManager::get()->GetFdEntity(path))){
if(NULL == (ent = FdManager::get()->GetFdEntity(path, static_cast<int>(fi->fh)))){
DPRN("could not find fd(file=%s)", path);
return -EIO;
}
@ -2078,7 +2078,7 @@ static int s3fs_release(const char* path, struct fuse_file_info* fi)
// check - for debug
if(debug){
if(NULL != (ent = FdManager::get()->GetFdEntity(path))){
if(NULL != (ent = FdManager::get()->GetFdEntity(path, static_cast<int>(fi->fh)))){
DPRNNN("Warning - file(%s),fd(%d) is still opened.", path, ent->GetFd());
}
}