Add mirror file logic for removing cache file

This commit is contained in:
Takeshi Nakatani 2016-07-03 03:37:08 +00:00 committed by Peter Watkins
parent ab4b92074c
commit 0f9428ad5a
4 changed files with 120 additions and 17 deletions

View File

@ -632,7 +632,7 @@ int FdEntity::FillFile(int fd, unsigned char byte, size_t size, off_t start)
// FdEntity methods // FdEntity methods
//------------------------------------------------ //------------------------------------------------
FdEntity::FdEntity(const char* tpath, const char* cpath) FdEntity::FdEntity(const char* tpath, const char* cpath)
: is_lock_init(false), refcnt(0), path(SAFESTRPTR(tpath)), cachepath(SAFESTRPTR(cpath)), : is_lock_init(false), refcnt(0), path(SAFESTRPTR(tpath)), cachepath(SAFESTRPTR(cpath)), mirrorpath(""),
fd(-1), pfile(NULL), is_modify(false), size_orgmeta(0), upload_id(""), mp_start(0), mp_size(0) fd(-1), pfile(NULL), is_modify(false), size_orgmeta(0), upload_id(""), mp_start(0), mp_size(0)
{ {
try{ try{
@ -671,15 +671,24 @@ void FdEntity::Clear(void)
S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str()); S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str());
} }
} }
fclose(pfile); if(pfile){
pfile = NULL; fclose(pfile);
fd = -1; pfile = NULL;
}
fd = -1;
if(!mirrorpath.empty()){
if(-1 == unlink(mirrorpath.c_str())){
S3FS_PRN_WARN("failed to remove mirror cache file(%s) by errno(%d).", mirrorpath.c_str(), errno);
}
mirrorpath.erase();
}
} }
pagelist.Init(0, false); pagelist.Init(0, false);
refcnt = 0; refcnt = 0;
path = ""; path = "";
cachepath = ""; cachepath = "";
is_modify = false; is_modify = false;
} }
void FdEntity::Close(void) void FdEntity::Close(void)
@ -699,9 +708,18 @@ void FdEntity::Close(void)
S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str()); S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str());
} }
} }
fclose(pfile); if(pfile){
pfile = NULL; fclose(pfile);
fd = -1; pfile = NULL;
}
fd = -1;
if(!mirrorpath.empty()){
if(-1 == unlink(mirrorpath.c_str())){
S3FS_PRN_WARN("failed to remove mirror cache file(%s) by errno(%d).", mirrorpath.c_str(), errno);
}
mirrorpath.erase();
}
} }
} }
} }
@ -717,6 +735,48 @@ int FdEntity::Dup(void)
return fd; return fd;
} }
//
// Open mirror file which is linked cache file.
//
int FdEntity::OpenMirrorFile(void)
{
if(cachepath.empty()){
S3FS_PRN_ERR("cache path is empty, why come here");
return -EIO;
}
// make tmporary directory
string bupdir;
if(!FdManager::MakeCachePath(NULL, bupdir, true, true)){
S3FS_PRN_ERR("could not make bup cache directory path or create it.");
return -EIO;
}
// make mirror file path
char szfile[NAME_MAX + 1];
if(NULL == tmpnam(szfile)){
S3FS_PRN_ERR("could not get temporary file name.");
return -EIO;
}
char* ppos = strrchr(szfile, '/');
++ppos;
mirrorpath = bupdir + "/" + ppos;
// link mirror file to cache file
if(-1 == link(cachepath.c_str(), mirrorpath.c_str())){
S3FS_PRN_ERR("could not link mirror file(%s) to cache file(%s) by errno(%d).", mirrorpath.c_str(), cachepath.c_str(), errno);
return -errno;
}
// open mirror file
int mirrorfd;
if(-1 == (mirrorfd = open(mirrorpath.c_str(), O_RDWR))){
S3FS_PRN_ERR("could not open mirror file(%s) by errno(%d).", mirrorpath.c_str(), errno);
return -errno;
}
return mirrorfd;
}
// [NOTE] // [NOTE]
// This method does not lock fdent_lock, because FdManager::fd_manager_lock // This method does not lock fdent_lock, because FdManager::fd_manager_lock
// is locked before calling. // is locked before calling.
@ -763,8 +823,9 @@ int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time)
// open cache and cache stat file, load page info. // open cache and cache stat file, load page info.
CacheFileStat cfstat(path.c_str()); CacheFileStat cfstat(path.c_str());
if(pagelist.Serialize(cfstat, false) && -1 != (fd = open(cachepath.c_str(), O_RDWR))){ // try to open cache file
// success to open cache file if(-1 != (fd = open(cachepath.c_str(), O_RDWR)) && pagelist.Serialize(cfstat, false)){
// succeed to open cache file and to load stats data
struct stat st; struct stat st;
memset(&st, 0, sizeof(struct stat)); memset(&st, 0, sizeof(struct stat));
if(-1 == fstat(fd, &st)){ if(-1 == fstat(fd, &st)){
@ -788,8 +849,9 @@ int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time)
is_truncate = true; is_truncate = true;
} }
} }
}else{ }else{
// could not load stat file or open file // could not open cache file or could not load stats data, so initialize it.
if(-1 == (fd = open(cachepath.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0600))){ if(-1 == (fd = open(cachepath.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0600))){
S3FS_PRN_ERR("failed to open file(%s). errno(%d)", cachepath.c_str(), errno); S3FS_PRN_ERR("failed to open file(%s). errno(%d)", cachepath.c_str(), errno);
return (0 == errno ? -EIO : -errno); return (0 == errno ? -EIO : -errno);
@ -804,6 +866,16 @@ int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time)
} }
} }
// open mirror file
int mirrorfd;
if(0 >= (mirrorfd = OpenMirrorFile())){
S3FS_PRN_ERR("failed to open mirror file linked cache file(%s).", cachepath.c_str());
return (0 == mirrorfd ? -EIO : mirrorfd);
}
// switch fd
close(fd);
fd = mirrorfd;
// make file pointer(for being same tmpfile) // make file pointer(for being same tmpfile)
if(NULL == (pfile = fdopen(fd, "wb"))){ if(NULL == (pfile = fdopen(fd, "wb"))){
S3FS_PRN_ERR("failed to get fileno(%s). errno(%d)", cachepath.c_str(), errno); S3FS_PRN_ERR("failed to get fileno(%s). errno(%d)", cachepath.c_str(), errno);
@ -1135,6 +1207,7 @@ int FdEntity::NoCacheLoadAndPost(off_t start, size_t size)
FdManager::DeleteCacheFile(path.c_str()); FdManager::DeleteCacheFile(path.c_str());
// cache file path does not use no more. // cache file path does not use no more.
cachepath.erase(); cachepath.erase();
mirrorpath.erase();
} }
// Change entity key in manager mapping // Change entity key in manager mapping
@ -1713,13 +1786,23 @@ int FdManager::DeleteCacheFile(const char* path)
return result; return result;
} }
bool FdManager::MakeCachePath(const char* path, string& cache_path, bool is_create_dir) bool FdManager::MakeCachePath(const char* path, string& cache_path, bool is_create_dir, bool is_mirror_path)
{ {
if(0 == FdManager::cache_dir.size()){ if(0 == FdManager::cache_dir.size()){
cache_path = ""; cache_path = "";
return true; return true;
} }
string resolved_path(FdManager::cache_dir + "/" + bucket);
string resolved_path(FdManager::cache_dir);
if(!is_mirror_path){
resolved_path += "/";
resolved_path += bucket;
}else{
resolved_path += "/.";
resolved_path += bucket;
resolved_path += ".mirror";
}
if(is_create_dir){ if(is_create_dir){
int result; int result;
if(0 != (result = mkdirp(resolved_path + mydirname(path), 0777))){ if(0 != (result = mkdirp(resolved_path + mydirname(path), 0777))){

View File

@ -117,6 +117,7 @@ class FdEntity
std::string path; // object path std::string path; // object path
std::string cachepath; // local cache file path std::string cachepath; // local cache file path
// (if this is empty, does not load/save pagelist.) // (if this is empty, does not load/save pagelist.)
std::string mirrorpath; // mirror file path to local cache file path
int fd; // file descriptor(tmp file or cache file) int fd; // file descriptor(tmp file or cache file)
FILE* pfile; // file pointer(tmp file or cache file) FILE* pfile; // file pointer(tmp file or cache file)
bool is_modify; // if file is changed, this flag is true bool is_modify; // if file is changed, this flag is true
@ -132,6 +133,7 @@ class FdEntity
static int FillFile(int fd, unsigned char byte, size_t size, off_t start); static int FillFile(int fd, unsigned char byte, size_t size, off_t start);
void Clear(void); void Clear(void);
int OpenMirrorFile(void);
bool SetAllStatus(bool is_loaded); // [NOTE] not locking bool SetAllStatus(bool is_loaded); // [NOTE] not locking
//bool SetAllStatusLoaded(void) { return SetAllStatus(true); } //bool SetAllStatusLoaded(void) { return SetAllStatus(true); }
bool SetAllStatusUnloaded(void) { return SetAllStatus(false); } bool SetAllStatusUnloaded(void) { return SetAllStatus(false); }
@ -202,7 +204,7 @@ class FdManager
static bool SetCacheDir(const char* dir); static bool SetCacheDir(const char* dir);
static bool IsCacheDir(void) { return (0 < FdManager::cache_dir.size()); } static bool IsCacheDir(void) { return (0 < FdManager::cache_dir.size()); }
static const char* GetCacheDir(void) { return FdManager::cache_dir.c_str(); } static const char* GetCacheDir(void) { return FdManager::cache_dir.c_str(); }
static bool MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir = true); static bool MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir = true, bool is_mirror_path = false);
static bool CheckCacheTopDir(void); static bool CheckCacheTopDir(void);
static bool MakeRandomTempPath(const char* path, std::string& tmppath); static bool MakeRandomTempPath(const char* path, std::string& tmppath);

View File

@ -545,6 +545,14 @@ int is_uid_inculde_group(uid_t uid, gid_t gid)
//------------------------------------------------------------------- //-------------------------------------------------------------------
// safe variant of dirname // safe variant of dirname
// dirname clobbers path so let it operate on a tmp copy // dirname clobbers path so let it operate on a tmp copy
string mydirname(const char* path)
{
if(!path || '\0' == path[0]){
return string("");
}
return mydirname(string(path));
}
string mydirname(string path) string mydirname(string path)
{ {
return string(dirname((char*)path.c_str())); return string(dirname((char*)path.c_str()));
@ -552,6 +560,14 @@ string mydirname(string path)
// safe variant of basename // safe variant of basename
// basename clobbers path so let it operate on a tmp copy // basename clobbers path so let it operate on a tmp copy
string mybasename(const char* path)
{
if(!path || '\0' == path[0]){
return string("");
}
return mybasename(string(path));
}
string mybasename(string path) string mybasename(string path)
{ {
return string(basename((char*)path.c_str())); return string(basename((char*)path.c_str()));

View File

@ -106,7 +106,9 @@ void free_mvnodes(MVNODE *head);
std::string get_username(uid_t uid); std::string get_username(uid_t uid);
int is_uid_inculde_group(uid_t uid, gid_t gid); int is_uid_inculde_group(uid_t uid, gid_t gid);
std::string mydirname(const char* path);
std::string mydirname(std::string path); std::string mydirname(std::string path);
std::string mybasename(const char* path);
std::string mybasename(std::string path); std::string mybasename(std::string path);
int mkdirp(const std::string& path, mode_t mode); int mkdirp(const std::string& path, mode_t mode);
bool check_exist_dir_permission(const char* dirpath); bool check_exist_dir_permission(const char* dirpath);