From e8a8019a7111e28d23a1322ec9e0e6bbf6ebaf20 Mon Sep 17 00:00:00 2001 From: Takeshi Nakatani Date: Sun, 3 Jul 2016 03:37:08 +0000 Subject: [PATCH] Add mirror file logic for removing cache file --- src/fdcache.cpp | 115 +++++++++++++++++++++++++++++++++++++++------- src/fdcache.h | 4 +- src/s3fs_util.cpp | 16 +++++++ src/s3fs_util.h | 2 + 4 files changed, 120 insertions(+), 17 deletions(-) diff --git a/src/fdcache.cpp b/src/fdcache.cpp index 881f0c7..92f6403 100644 --- a/src/fdcache.cpp +++ b/src/fdcache.cpp @@ -632,7 +632,7 @@ int FdEntity::FillFile(int fd, unsigned char byte, size_t size, off_t start) // FdEntity methods //------------------------------------------------ FdEntity::FdEntity(const char* tpath, const char* cpath) - : is_lock_init(false), refcnt(0), path(SAFESTRPTR(tpath)), cachepath(SAFESTRPTR(cpath)), + : is_lock_init(false), refcnt(0), path(SAFESTRPTR(tpath)), cachepath(SAFESTRPTR(cpath)), mirrorpath(""), fd(-1), pfile(NULL), is_modify(false), size_orgmeta(0), upload_id(""), mp_start(0), mp_size(0) { try{ @@ -671,15 +671,24 @@ void FdEntity::Clear(void) S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str()); } } - fclose(pfile); - pfile = NULL; - fd = -1; + if(pfile){ + fclose(pfile); + pfile = NULL; + } + fd = -1; + + if(!mirrorpath.empty()){ + if(-1 == unlink(mirrorpath.c_str())){ + S3FS_PRN_WARN("failed to remove mirror cache file(%s) by errno(%d).", mirrorpath.c_str(), errno); + } + mirrorpath.erase(); + } } pagelist.Init(0, false); - refcnt = 0; - path = ""; - cachepath = ""; - is_modify = false; + refcnt = 0; + path = ""; + cachepath = ""; + is_modify = false; } void FdEntity::Close(void) @@ -699,9 +708,18 @@ void FdEntity::Close(void) S3FS_PRN_WARN("failed to save cache stat file(%s).", path.c_str()); } } - fclose(pfile); - pfile = NULL; - fd = -1; + if(pfile){ + fclose(pfile); + pfile = NULL; + } + fd = -1; + + if(!mirrorpath.empty()){ + if(-1 == unlink(mirrorpath.c_str())){ + S3FS_PRN_WARN("failed to remove mirror cache file(%s) by errno(%d).", mirrorpath.c_str(), errno); + } + mirrorpath.erase(); + } } } } @@ -717,6 +735,48 @@ int FdEntity::Dup(void) return fd; } +// +// Open mirror file which is linked cache file. +// +int FdEntity::OpenMirrorFile(void) +{ + if(cachepath.empty()){ + S3FS_PRN_ERR("cache path is empty, why come here"); + return -EIO; + } + + // make tmporary directory + string bupdir; + if(!FdManager::MakeCachePath(NULL, bupdir, true, true)){ + S3FS_PRN_ERR("could not make bup cache directory path or create it."); + return -EIO; + } + + // make mirror file path + char szfile[NAME_MAX + 1]; + if(NULL == tmpnam(szfile)){ + S3FS_PRN_ERR("could not get temporary file name."); + return -EIO; + } + char* ppos = strrchr(szfile, '/'); + ++ppos; + mirrorpath = bupdir + "/" + ppos; + + // link mirror file to cache file + if(-1 == link(cachepath.c_str(), mirrorpath.c_str())){ + S3FS_PRN_ERR("could not link mirror file(%s) to cache file(%s) by errno(%d).", mirrorpath.c_str(), cachepath.c_str(), errno); + return -errno; + } + + // open mirror file + int mirrorfd; + if(-1 == (mirrorfd = open(mirrorpath.c_str(), O_RDWR))){ + S3FS_PRN_ERR("could not open mirror file(%s) by errno(%d).", mirrorpath.c_str(), errno); + return -errno; + } + return mirrorfd; +} + // [NOTE] // This method does not lock fdent_lock, because FdManager::fd_manager_lock // is locked before calling. @@ -763,8 +823,9 @@ int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time) // open cache and cache stat file, load page info. CacheFileStat cfstat(path.c_str()); - if(pagelist.Serialize(cfstat, false) && -1 != (fd = open(cachepath.c_str(), O_RDWR))){ - // success to open cache file + // try to open cache file + if(-1 != (fd = open(cachepath.c_str(), O_RDWR)) && pagelist.Serialize(cfstat, false)){ + // succeed to open cache file and to load stats data struct stat st; memset(&st, 0, sizeof(struct stat)); if(-1 == fstat(fd, &st)){ @@ -788,8 +849,9 @@ int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time) is_truncate = true; } } + }else{ - // could not load stat file or open file + // could not open cache file or could not load stats data, so initialize it. if(-1 == (fd = open(cachepath.c_str(), O_CREAT|O_RDWR|O_TRUNC, 0600))){ S3FS_PRN_ERR("failed to open file(%s). errno(%d)", cachepath.c_str(), errno); return (0 == errno ? -EIO : -errno); @@ -804,6 +866,16 @@ int FdEntity::Open(headers_t* pmeta, ssize_t size, time_t time) } } + // open mirror file + int mirrorfd; + if(0 >= (mirrorfd = OpenMirrorFile())){ + S3FS_PRN_ERR("failed to open mirror file linked cache file(%s).", cachepath.c_str()); + return (0 == mirrorfd ? -EIO : mirrorfd); + } + // switch fd + close(fd); + fd = mirrorfd; + // make file pointer(for being same tmpfile) if(NULL == (pfile = fdopen(fd, "wb"))){ S3FS_PRN_ERR("failed to get fileno(%s). errno(%d)", cachepath.c_str(), errno); @@ -1135,6 +1207,7 @@ int FdEntity::NoCacheLoadAndPost(off_t start, size_t size) FdManager::DeleteCacheFile(path.c_str()); // cache file path does not use no more. cachepath.erase(); + mirrorpath.erase(); } // Change entity key in manager mapping @@ -1713,13 +1786,23 @@ int FdManager::DeleteCacheFile(const char* path) return result; } -bool FdManager::MakeCachePath(const char* path, string& cache_path, bool is_create_dir) +bool FdManager::MakeCachePath(const char* path, string& cache_path, bool is_create_dir, bool is_mirror_path) { if(0 == FdManager::cache_dir.size()){ cache_path = ""; return true; } - string resolved_path(FdManager::cache_dir + "/" + bucket); + + string resolved_path(FdManager::cache_dir); + if(!is_mirror_path){ + resolved_path += "/"; + resolved_path += bucket; + }else{ + resolved_path += "/."; + resolved_path += bucket; + resolved_path += ".mirror"; + } + if(is_create_dir){ int result; if(0 != (result = mkdirp(resolved_path + mydirname(path), 0777))){ diff --git a/src/fdcache.h b/src/fdcache.h index 423c5b5..2c5c9a3 100644 --- a/src/fdcache.h +++ b/src/fdcache.h @@ -117,6 +117,7 @@ class FdEntity std::string path; // object path std::string cachepath; // local cache file path // (if this is empty, does not load/save pagelist.) + std::string mirrorpath; // mirror file path to local cache file path int fd; // file descriptor(tmp file or cache file) FILE* pfile; // file pointer(tmp file or cache file) bool is_modify; // if file is changed, this flag is true @@ -132,6 +133,7 @@ class FdEntity static int FillFile(int fd, unsigned char byte, size_t size, off_t start); void Clear(void); + int OpenMirrorFile(void); bool SetAllStatus(bool is_loaded); // [NOTE] not locking //bool SetAllStatusLoaded(void) { return SetAllStatus(true); } bool SetAllStatusUnloaded(void) { return SetAllStatus(false); } @@ -202,7 +204,7 @@ class FdManager static bool SetCacheDir(const char* dir); static bool IsCacheDir(void) { return (0 < FdManager::cache_dir.size()); } static const char* GetCacheDir(void) { return FdManager::cache_dir.c_str(); } - static bool MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir = true); + static bool MakeCachePath(const char* path, std::string& cache_path, bool is_create_dir = true, bool is_mirror_path = false); static bool CheckCacheTopDir(void); static bool MakeRandomTempPath(const char* path, std::string& tmppath); diff --git a/src/s3fs_util.cpp b/src/s3fs_util.cpp index 2999da9..56ac6e2 100644 --- a/src/s3fs_util.cpp +++ b/src/s3fs_util.cpp @@ -545,6 +545,14 @@ int is_uid_inculde_group(uid_t uid, gid_t gid) //------------------------------------------------------------------- // safe variant of dirname // dirname clobbers path so let it operate on a tmp copy +string mydirname(const char* path) +{ + if(!path || '\0' == path[0]){ + return string(""); + } + return mydirname(string(path)); +} + string mydirname(string path) { return string(dirname((char*)path.c_str())); @@ -552,6 +560,14 @@ string mydirname(string path) // safe variant of basename // basename clobbers path so let it operate on a tmp copy +string mybasename(const char* path) +{ + if(!path || '\0' == path[0]){ + return string(""); + } + return mybasename(string(path)); +} + string mybasename(string path) { return string(basename((char*)path.c_str())); diff --git a/src/s3fs_util.h b/src/s3fs_util.h index d4d8ac5..a105c80 100644 --- a/src/s3fs_util.h +++ b/src/s3fs_util.h @@ -106,7 +106,9 @@ void free_mvnodes(MVNODE *head); std::string get_username(uid_t uid); int is_uid_inculde_group(uid_t uid, gid_t gid); +std::string mydirname(const char* path); std::string mydirname(std::string path); +std::string mybasename(const char* path); std::string mybasename(std::string path); int mkdirp(const std::string& path, mode_t mode); bool check_exist_dir_permission(const char* dirpath);